From a689ae9e5ff46081db05c9609730df00b90c90eb Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 12 Mar 2024 10:04:56 +0900 Subject: [PATCH 001/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4763) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 5e3a3077f5d..1c384183fbd 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.03 && cd - + cd ../amrex && git checkout --detach 155f1f4c3f4fa03528d38d2769beb4b35b38042c && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 2f0c7b607bb..138cf90534f 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.03 +branch = 155f1f4c3f4fa03528d38d2769beb4b35b38042c [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index d6ac81c5607..daeef409dc8 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.03 +branch = 155f1f4c3f4fa03528d38d2769beb4b35b38042c [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index a1045457ec0..54b1cc45322 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.03" +set(WarpX_amrex_branch "155f1f4c3f4fa03528d38d2769beb4b35b38042c" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 6360da84276..eaa0fc016cd 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.03" +set(WarpX_pyamrex_branch "7cd4a4220bc3fc523cd0d78f7320909aa2feb47a" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index f556dd37e76..1bf6e96d236 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.03 && cd - +cd amrex && git checkout --detach 155f1f4c3f4fa03528d38d2769beb4b35b38042c && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 7317aafb9d99b74d077c5bcc1bae3af29f31d9fd Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Tue, 12 Mar 2024 06:54:21 -0700 Subject: [PATCH 002/121] clean up (#4761) --- Source/Particles/PhysicalParticleContainer.H | 8 +--- .../Particles/PhysicalParticleContainer.cpp | 40 ++++++++----------- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index b77c1147a15..286d0675da6 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -226,13 +226,7 @@ public: amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, amrex::Real t_lab = 0.) const; - void AddGaussianBeam ( - PlasmaInjector const& plasma_injector, - amrex::Real x_m, amrex::Real y_m, amrex::Real z_m, - amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, - amrex::Real x_cut, amrex::Real y_cut, amrex::Real z_cut, - amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order, - amrex::Real focal_distance); + void AddGaussianBeam (PlasmaInjector const& plasma_injector); /** Load a particle beam from an external file * @param[in] the PlasmaInjector instance holding the input parameters diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1aa534c871b..160eac0d19c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -521,14 +521,22 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame ( } void -PhysicalParticleContainer::AddGaussianBeam ( - PlasmaInjector const& plasma_injector, - const Real x_m, const Real y_m, const Real z_m, - const Real x_rms, const Real y_rms, const Real z_rms, - const Real x_cut, const Real y_cut, const Real z_cut, - const Real q_tot, long npart, - const int do_symmetrize, - const int symmetrization_order, const Real focal_distance) { +PhysicalParticleContainer::AddGaussianBeam (PlasmaInjector const& plasma_injector){ + + const Real x_m = plasma_injector.x_m; + const Real y_m = plasma_injector.y_m; + const Real z_m = plasma_injector.z_m; + const Real x_rms = plasma_injector.x_rms; + const Real y_rms = plasma_injector.y_rms; + const Real z_rms = plasma_injector.z_rms; + const Real x_cut = plasma_injector.x_cut; + const Real y_cut = plasma_injector.y_cut; + const Real z_cut = plasma_injector.z_cut; + const Real q_tot = plasma_injector.q_tot; + long npart = plasma_injector.npart; + const int do_symmetrize = plasma_injector.do_symmetrize; + const int symmetrization_order = plasma_injector.symmetrization_order; + const Real focal_distance = plasma_injector.focal_distance; // Declare temporary vectors on the CPU Gpu::HostVector particle_x; @@ -918,21 +926,7 @@ PhysicalParticleContainer::AddParticles (int lev) } if (plasma_injector->gaussian_beam) { - AddGaussianBeam(*plasma_injector, - plasma_injector->x_m, - plasma_injector->y_m, - plasma_injector->z_m, - plasma_injector->x_rms, - plasma_injector->y_rms, - plasma_injector->z_rms, - plasma_injector->x_cut, - plasma_injector->y_cut, - plasma_injector->z_cut, - plasma_injector->q_tot, - plasma_injector->npart, - plasma_injector->do_symmetrize, - plasma_injector->symmetrization_order, - plasma_injector->focal_distance); + AddGaussianBeam(*plasma_injector); } if (plasma_injector->external_file) { From 966efe04ccfab2ef3369af86a315d5b90b43dec3 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 13 Mar 2024 17:21:41 +0100 Subject: [PATCH 003/121] remove superfluous include directives (#4770) --- Source/Initialization/WarpXAMReXInit.cpp | 2 -- Source/ablastr/utils/text/StreamUtils.cpp | 1 - 2 files changed, 3 deletions(-) diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index d1dd0bbe90b..8cfe78c740f 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -11,8 +11,6 @@ #include #include -#include - namespace { /** Overwrite defaults in AMReX Inputs * diff --git a/Source/ablastr/utils/text/StreamUtils.cpp b/Source/ablastr/utils/text/StreamUtils.cpp index 965beb179ba..541ef300f7c 100644 --- a/Source/ablastr/utils/text/StreamUtils.cpp +++ b/Source/ablastr/utils/text/StreamUtils.cpp @@ -7,7 +7,6 @@ #include "StreamUtils.H" -#include #include void From 41983ae761c0da0dc6220c25b547f35d4ce3bb9f Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Wed, 13 Mar 2024 21:48:53 -0700 Subject: [PATCH 004/121] Adding hyper-resistivity to generalized ohms law hybrid solver. (#4772) * Adding hyper-resistivity to generalized ohms law hybrid solver. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestions from code review Style changes to clean up Laplacian operators. Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Removed extra arguments for triggering calculation of hyper_resistivity. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Python/pywarpx/picmi.py | 5 +- .../CartesianYeeAlgorithm.H | 58 +++++++++++++++++++ .../CylindricalYeeAlgorithm.H | 33 +++++++++++ .../HybridPICModel/HybridPICModel.H | 3 + .../HybridPICModel/HybridPICModel.cpp | 2 + .../HybridPICSolveE.cpp | 50 +++++++++++++--- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 3 +- 7 files changed, 144 insertions(+), 10 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 959db6c701d..59ca1ec789f 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1142,7 +1142,8 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): Function of space and time specifying external (non-plasma) currents. """ def __init__(self, grid, Te=None, n0=None, gamma=None, - n_floor=None, plasma_resistivity=None, substeps=None, + n_floor=None, plasma_resistivity=None, + plasma_hyper_resistivity=None, substeps=None, Jx_external_function=None, Jy_external_function=None, Jz_external_function=None, **kw): self.grid = grid @@ -1153,6 +1154,7 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.gamma = gamma self.n_floor = n_floor self.plasma_resistivity = plasma_resistivity + self.plasma_hyper_resistivity = plasma_hyper_resistivity self.substeps = substeps @@ -1187,6 +1189,7 @@ def solver_initialize_inputs(self): 'plasma_resistivity(rho,J)', pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) + pywarpx.hybridpicmodel.plasma_hyper_resistivity = self.plasma_hyper_resistivity pywarpx.hybridpicmodel.substeps = self.substeps pywarpx.hybridpicmodel.__setattr__( 'Jx_external_grid_function(x,y,z,t)', diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H index b762530e1a3..c4978287aec 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H @@ -100,6 +100,25 @@ struct CartesianYeeAlgorithm { #endif } + /** + * Perform second derivative along x on a cell-centered grid, from a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dxx ( + T_Field const& F, + amrex::Real const * const coefs_x, int const /*n_coefs_x*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; +#if (defined WARPX_DIM_1D_Z) + amrex::ignore_unused(F, coefs_x, i, j, k, ncomp); + return 0._rt; // 1D Cartesian: derivative along x is 0 +#else + amrex::Real const inv_dx2 = coefs_x[0]*coefs_x[0]; + return inv_dx2*( F(i-1,j,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i+1,j,k,ncomp) ); +#endif + } + /** * Perform derivative along y on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -147,6 +166,25 @@ struct CartesianYeeAlgorithm { #endif } + /** + * Perform derivative along y on a nodal grid, from a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dyy ( + T_Field const& F, + amrex::Real const * const coefs_y, int const /*n_coefs_y*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; +#if defined WARPX_DIM_3D + Real const inv_dy2 = coefs_y[0]*coefs_y[0]; + return inv_dy2*( F(i,j-1,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j+1,k,ncomp) ); +#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) + amrex::ignore_unused(F, coefs_y, i, j, k, ncomp); + return 0._rt; // 1D and 2D Cartesian: derivative along y is 0 +#endif + } + /** * Perform derivative along z on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -186,6 +224,26 @@ struct CartesianYeeAlgorithm { #endif } + /** + * Perform second derivative along z on a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dzz ( + T_Field const& F, + amrex::Real const * const coefs_z, int const /*n_coefs_z*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; + Real const inv_dz2 = coefs_z[0]*coefs_z[0]; +#if defined WARPX_DIM_3D + return inv_dz2*( F(i,j,k-1,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j,k+1,ncomp) ); +#elif (defined WARPX_DIM_XZ) + return inv_dz2*( F(i,j-1,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j+1,k,ncomp) ); +#elif (defined WARPX_DIM_1D_Z) + return inv_dz2*( F(i-1,j,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i+1,j,k,ncomp) ); +#endif + } + }; #endif // WARPX_FINITE_DIFFERENCE_ALGORITHM_CARTESIAN_YEE_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H index e49995557a0..436f0a83f31 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H @@ -134,6 +134,24 @@ struct CylindricalYeeAlgorithm { return inv_dr*( F(i,j,k,comp) - F(i-1,j,k,comp) ); } + /** Applies the differential operator `1/r * d(r * dF/dr)/dr`, + * where `F` is on a *cell-centered* or a nodal grid in `r` + * The input parameter `r` is given at the cell-centered position */ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dr_rDr_over_r ( + T_Field const& F, + amrex::Real const r, amrex::Real const dr, + amrex::Real const * const coefs_r, int const /*n_coefs_r*/, + int const i, int const j, int const k, int const comp ) { + + using namespace amrex; + + Real const inv_dr2 = coefs_r[0]*coefs_r[0]; + return 1._rt/r * inv_dr2*( (r+0.5_rt*dr)*(F(i+1,j,k,comp) - F(i,j,k,comp)) + - (r-0.5_rt*dr)*(F(i,j,k,comp) - F(i-1,j,k,comp)) ); + } + /** * Perform derivative along z on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -162,6 +180,21 @@ struct CylindricalYeeAlgorithm { return inv_dz*( F(i,j,k,comp) - F(i,j-1,k,comp) ); } + /** + * Perform second derivative along z on a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dzz ( + T_Field const& F, + amrex::Real const * const coefs_z, int const /*n_coefs_z*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; + Real const inv_dz2 = coefs_z[0]*coefs_z[0]; + + return inv_dz2*( F(i,j-1,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j+1,k,ncomp) ); + } + }; #endif // WARPX_FINITE_DIFFERENCE_ALGORITHM_CYLINDRICAL_YEE_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 23ef49b58cb..5e4c6382bfc 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -175,6 +175,9 @@ public: amrex::ParserExecutor<2> m_eta; bool m_resistivity_has_J_dependence = false; + /** Plasma hyper-resisitivity */ + amrex::Real m_eta_h = 0.0; + /** External current */ std::string m_Jx_ext_grid_function = "0.0"; std::string m_Jy_ext_grid_function = "0.0"; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8c2b5a0707a..e8330242fe8 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -40,6 +40,8 @@ void HybridPICModel::ReadParameters () pp_hybrid.query("plasma_resistivity(rho,J)", m_eta_expression); utils::parser::queryWithParser(pp_hybrid, "n_floor", m_n_floor); + utils::parser::queryWithParser(pp_hybrid, "plasma_hyper_resistivity", m_eta_h); + // convert electron temperature from eV to J m_elec_temp *= PhysConst::q_e; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 066e88b13c0..456c542a534 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -375,7 +375,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, - const bool include_resistivity_term ) + const bool include_resistivity_term) { // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -432,9 +432,12 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; + const auto eta_h = hybrid_model->m_eta_h; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; + const bool include_hyper_resistivity_term = (eta_h > 0.0) && include_resistivity_term; + // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations amrex::GpuArray const& Er_stag = hybrid_model->Ex_IndexType; @@ -612,6 +615,14 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val, jtot_val) * Jr(i, j, 0); } + + if (include_hyper_resistivity_term) { + // r on cell-centered point (Jr is cell-centered in r) + Real const r = rmin + (i + 0.5_rt)*dr; + + auto nabla2Jr = T_Algo::Dr_rDr_over_r(Jr, r, dr, coefs_r, n_coefs_r, i, j, 0, 0); + Er(i, j, 0) -= eta_h * nabla2Jr; + } }, // Et calculation @@ -655,16 +666,18 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val, jtot_val) * Jt(i, j, 0); } + + // Note: Hyper-resisitivity should be revisited here when modal decomposition is implemented }, // Ez calculation - [=] AMREX_GPU_DEVICE (int i, int j, int k){ + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries if (lz(i,j,0) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, 0, 0); // Interpolate current to appropriate staggering to match E field Real jtot_val = 0._rt; @@ -679,15 +692,20 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k, 0); + auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, 0, 0); // interpolate the nodal neE values to the Yee grid - auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, k, 2); + auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, 0, 2); - Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; + Ez(i, j, 0) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } + if (include_resistivity_term) { Ez(i, j, 0) += eta(rho_val, jtot_val) * Jz(i, j, 0); } + + if (include_hyper_resistivity_term) { + auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, 0, 0); + Ez(i, j, 0) -= eta_h * nabla2Jz; + } } ); @@ -726,9 +744,12 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; + const auto eta_h = hybrid_model->m_eta_h; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; + const bool include_hyper_resistivity_term = (eta_h > 0.) && include_resistivity_term; + // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations amrex::GpuArray const& Ex_stag = hybrid_model->Ex_IndexType; @@ -904,6 +925,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val, jtot_val) * Jx(i, j, k); } + + if (include_hyper_resistivity_term) { + auto nabla2Jx = T_Algo::Dxx(Jx, coefs_x, n_coefs_x, i, j, k); + Ex(i, j, k) -= eta_h * nabla2Jx; + } }, // Ey calculation @@ -943,6 +969,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val, jtot_val) * Jy(i, j, k); } + + if (include_hyper_resistivity_term) { + auto nabla2Jy = T_Algo::Dyy(Jy, coefs_y, n_coefs_y, i, j, k); + Ey(i, j, k) -= eta_h * nabla2Jy; + } }, // Ez calculation @@ -976,6 +1007,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } + + if (include_hyper_resistivity_term) { + auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, k); + Ez(i, j, k) -= eta_h * nabla2Jz; + } } ); diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index ae10ae1e19a..4dbe10c4e5a 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -154,8 +154,7 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, rho_fp, m_edge_lengths, - false + Efield_fp, current_fp_temp, Bfield_fp, rho_fp, m_edge_lengths, false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); From 2630be83f210d32fed4a48bbb28ec3db372cf433 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 14 Mar 2024 21:43:05 +0900 Subject: [PATCH 005/121] Doc: Conda `-y` Install Block (#4776) * Doc: Update libmamba Usage in Conda * Fix: The configuration key changed slightly. * Cosmetic: Adding `-y` ensures one can execute the whole block at once. * `solver` is the new (conda>=23) key! * Fix: Double backticks for verbatim --- Docs/source/install/dependencies.rst | 4 ++-- Docs/source/install/users.rst | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 94b2be78bec..4f3544d5c0d 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -63,8 +63,8 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda update -n base conda - conda install -n base conda-libmamba-solver + conda update -y -n base conda + conda install -y -n base conda-libmamba-solver conda config --set solver libmamba We recommend to deactivate that conda self-activates its ``base`` environment. diff --git a/Docs/source/install/users.rst b/Docs/source/install/users.rst index b7893e248c0..e56d2d8ac43 100644 --- a/Docs/source/install/users.rst +++ b/Docs/source/install/users.rst @@ -45,12 +45,12 @@ A package for WarpX is available via the `Conda `_ package man .. tip:: - We recommend to configure your conda to use the faster `libmamba` `dependency solver `__. + We recommend to configure your conda to use the faster ``libmamba`` `dependency solver `__. .. code-block:: bash - conda update -n base conda - conda install -n base conda-libmamba-solver + conda update -y -n base conda + conda install -y -n base conda-libmamba-solver conda config --set solver libmamba We recommend to deactivate that conda self-activates its ``base`` environment. From 467598cb56a1bdc886c8c2134fe9b33e748f9de6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 15 Mar 2024 00:50:28 +0900 Subject: [PATCH 006/121] Zenodo: Add Justin & Kale as Co-Authors (#4760) * Zenodo: Add Justin & Kale as Co-Authors Adding Justin and Kale as a co-author of WarpX. Thank you for your contributions! :sparkle: * Update Order in Kale's Affiliation * Update Justin MN --- .zenodo.json | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index bbba267def9..5431e40c8bb 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -20,6 +20,11 @@ "name": "Andriyash, Igor", "orcid": "0000-0003-0313-4496" }, + { + "affiliation": "Lawrence Livermore National Laboratory", + "name": "Angus, Justin Ray", + "orcid": "0000-0003-1474-0002" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Belkin, Daniel", @@ -155,7 +160,7 @@ }, { "affiliation": "Lawrence Berkeley National Laboratory", - "name": "Sandberg, Ryan T.", + "name": "Sandberg, Ryan Thor", "orcid": "0000-0001-7680-8733" }, { @@ -163,11 +168,21 @@ "name": "Scherpelz, Peter", "orcid": "0000-0001-8185-3387" }, + { + "affiliation": "Laboratory for Laser Energetics, University of Rochester", + "name": "Weichman, Kale", + "orcid": "0000-0002-3487-7922" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Yang, Eloise", "orcid": "0000-0002-9319-4216" }, + { + "affiliation": "LIDYL, CEA-Universit\u00e9 Paris-Saclay, CEA Saclay", + "name": "Zaim, Ne\u00efl", + "orcid": "0000-0003-0313-4496" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Zhang, Weiqun", @@ -187,11 +202,6 @@ "affiliation": "Lawrence Berkeley National Laboratory", "name": "Zoni, Edoardo", "orcid": "0000-0001-5662-4646" - }, - { - "affiliation": "LIDYL, CEA-Universit\u00e9 Paris-Saclay, CEA Saclay", - "name": "Zaim, Ne\u00efl", - "orcid": "0000-0003-0313-4496" } ], "contributors": [ From 5f96976b18f19fe748e021a3f4ecf6999c68c038 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 14 Mar 2024 11:33:36 -0700 Subject: [PATCH 007/121] Compute electrostatic fields at the beginning of EM simulations, if potential is specified (#4723) * Compute electrostatic fields at the beginning of EM simulations, if phi is specified * Fix tests * Add documentation * Update test to include initial electrostatic field * Avoid issue in CI * Set the potential to be uniform * Update checksum * Update automated test * Update checksum * Add warning message * Update Source/WarpX.cpp * Update Source/WarpX.cpp * Update Source/WarpX.cpp Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/parameters.rst | 17 ++++++--- .../Tests/embedded_boundary_cube/inputs_2d | 6 +++- .../Tests/embedded_boundary_cube/inputs_3d | 5 +++ Examples/Tests/point_of_contact_EB/inputs_3d | 6 ---- Examples/Tests/point_of_contact_EB/inputs_rz | 6 ---- .../embedded_boundary_cube.json | 15 ++++---- .../embedded_boundary_cube_2d.json | 10 +++--- .../embedded_boundary_cube_macroscopic.json | 10 +++--- Regression/WarpX-tests.ini | 6 ++-- Source/FieldSolver/ElectrostaticSolver.cpp | 3 +- Source/WarpX.H | 1 + Source/WarpX.cpp | 35 +++++++++++++++---- 12 files changed, 74 insertions(+), 46 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 0c5fe85973f..a615c9c8a87 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -395,12 +395,18 @@ Domain Boundary Conditions * ``damped``: This is the recommended option in the moving direction when using the spectral solver with moving window (currently only supported along z). This boundary condition applies a damping factor to the electric and magnetic fields in the outer half of the guard cells, using a sine squared profile. As the spectral solver is by nature periodic, the damping prevents fields from wrapping around to the other end of the domain when the periodicity is not desired. This boundary condition is only valid when using the spectral solver. * ``pec``: This option can be used to set a Perfect Electric Conductor at the simulation boundary. Please see the :ref:`PEC theory section ` for more details. Note that PEC boundary is invalid at `r=0` for the RZ solver. Please use ``none`` option. This boundary condition does not work with the spectral solver. - If an electrostatic field solve is used the boundary potentials can also be set through ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`). * ``none``: No boundary condition is applied to the fields with the electromagnetic solver. This option must be used for the RZ-solver at `r=0`. * ``neumann``: For the electrostatic solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. +* ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) + Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers + (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential + in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), + setting these variables will trigger an electrostatic solve at ``t=0``, to compute the initial + electric field produced by the boundaries. + * ``boundary.particle_lo`` and ``boundary.particle_hi`` (`2 strings` for 2D, `3 strings` for 3D, `absorbing` by default) Options are: @@ -473,9 +479,12 @@ Embedded Boundary Conditions the interior of the embeddded boundary is where the function value is positive. * ``warpx.eb_potential(x,y,z,t)`` (`string`) - Only used when ``warpx.do_electrostatic=labframe``. Gives the value of - the electric potential at the surface of the embedded boundary, - as a function of `x`, `y`, `z` and time. This function is also evaluated + Gives the value of the electric potential at the surface of the embedded boundary, + as a function of `x`, `y`, `z` and `t`. With electrostatic solvers (i.e., with + ``warpx.do_electrostatic = ...``), this is used in order to compute the potential + in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), + setting this variable will trigger an electrostatic solve at ``t=0``, to compute the initial + electric field produced by the boundaries. Note that this function is also evaluated inside the embedded boundary. For this reason, it is important to define this function in such a way that it is constant inside the embedded boundary. diff --git a/Examples/Tests/embedded_boundary_cube/inputs_2d b/Examples/Tests/embedded_boundary_cube/inputs_2d index 476992b360d..372e0dc0340 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_2d +++ b/Examples/Tests/embedded_boundary_cube/inputs_2d @@ -15,10 +15,14 @@ my_constants.xmin = -0.5 my_constants.zmin = -0.5 my_constants.xmax = 0.5 my_constants.zmax = 0.5 -# Alternatively one could use parser to build EB # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. warpx.eb_implicit_function = "max(max(x+xmin,-(x+xmax)), max(z+zmin,-(z+zmax)))" +# To test the initial electrostatic solver, we also add uniform potential +# Since, the potential is uniform here, it should not modify the fields, so this mainly +# tests that calling the electrostatic solver does not raise any error/crash +warpx.eb_potential(x,y,z,t) = "1" + warpx.B_ext_grid_init_style = parse_B_ext_grid_function warpx.E_ext_grid_init_style = parse_E_ext_grid_function diff --git a/Examples/Tests/embedded_boundary_cube/inputs_3d b/Examples/Tests/embedded_boundary_cube/inputs_3d index a98cd0b75ac..61eb1192e04 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_3d +++ b/Examples/Tests/embedded_boundary_cube/inputs_3d @@ -19,6 +19,11 @@ eb2.box_has_fluid_inside = true # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. # warpx.eb_implicit_function = "max(max(max(x-0.5,-0.5-x), max(y-0.5,-0.5-y)), max(z-0.5,-0.5-z))" +# To test the initial electrostatic solver, we also add uniform potential +# Since, the potential is uniform here, it should not modify the fields, so this mainly +# tests that calling the electrostatic solver does not raise any error/crash +warpx.eb_potential(x,y,z,t) = "1" + warpx.B_ext_grid_init_style = parse_B_ext_grid_function my_constants.m = 0 my_constants.n = 1 diff --git a/Examples/Tests/point_of_contact_EB/inputs_3d b/Examples/Tests/point_of_contact_EB/inputs_3d index 005733487e5..8b190c056ff 100644 --- a/Examples/Tests/point_of_contact_EB/inputs_3d +++ b/Examples/Tests/point_of_contact_EB/inputs_3d @@ -12,12 +12,6 @@ geometry.prob_hi = 0.26 0.26 0.26 boundary.field_lo = pec pec pec boundary.field_hi = pec pec pec -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 warpx.const_dt = 1e-10 warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.2**2)" diff --git a/Examples/Tests/point_of_contact_EB/inputs_rz b/Examples/Tests/point_of_contact_EB/inputs_rz index a492c1f705a..169a58adb94 100644 --- a/Examples/Tests/point_of_contact_EB/inputs_rz +++ b/Examples/Tests/point_of_contact_EB/inputs_rz @@ -12,12 +12,6 @@ geometry.prob_hi = 0.26 0.26 boundary.field_lo = none periodic boundary.field_hi = pec periodic -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 warpx.const_dt = 1e-10 warpx.eb_implicit_function = "-(x**2 -0.2**2)" diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json index 69fffed2c6a..d0edafb9f0e 100644 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json +++ b/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json @@ -1,11 +1,10 @@ { "lev=0": { - "Bx": 0.0, - "By": 0.0066283741197868335, - "Bz": 0.006628374119786833, - "Ex": 5102618.47115243, - "Ey": 0.0, - "Ez": 0.0 + "Bx": 3.769898030127477e-18, + "By": 0.006628374119786834, + "Bz": 0.006628374119786834, + "Ex": 5102618.4711524295, + "Ey": 6.323755130400527e-05, + "Ez": 6.323755130400527e-05 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json index 37a82967a23..a3e609bd9a9 100644 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json +++ b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 9.263694545408502e-05, - "By": 0.00031905198933489135, - "Bz": 7.328424783762596e-05, - "Ex": 8553.90669805307, + "Bx": 9.263694545408503e-05, + "By": 0.00031905198933489145, + "Bz": 7.328424783762594e-05, + "Ex": 8553.906698053046, "Ey": 60867.04830538045, - "Ez": 0.0 + "Ez": 8.439422682267567e-07 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json index 453b83f5e31..c759f36ac5d 100644 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json +++ b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 0.0, - "By": 0.005101824310293575, + "Bx": 3.898540288712651e-18, + "By": 0.005101824310293573, "Bz": 0.005101824310293573, - "Ex": 4414725.184731115, - "Ey": 0.0, - "Ez": 0.0 + "Ex": 4414725.184731114, + "Ey": 6.323754812712063e-05, + "Ez": 6.323754812712063e-05 } } \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index daeef409dc8..56a7fbc8f10 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -605,7 +605,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sph [embedded_boundary_cube] buildDir = . inputFile = Examples/Tests/embedded_boundary_cube/inputs_3d -runtime_params = +runtime_params = warpx.abort_on_warning_threshold = medium dim = 3 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON @@ -622,7 +622,7 @@ analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_cube_2d] buildDir = . inputFile = Examples/Tests/embedded_boundary_cube/inputs_2d -runtime_params = +runtime_params = warpx.abort_on_warning_threshold = medium dim = 2 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON @@ -639,7 +639,7 @@ analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py [embedded_boundary_cube_macroscopic] buildDir = . inputFile = Examples/Tests/embedded_boundary_cube/inputs_3d -runtime_params = algo.em_solver_medium=macroscopic macroscopic.epsilon=1.5*8.8541878128e-12 macroscopic.sigma=0 macroscopic.mu=1.25663706212e-06 +runtime_params = algo.em_solver_medium=macroscopic macroscopic.epsilon=1.5*8.8541878128e-12 macroscopic.sigma=0 macroscopic.mu=1.25663706212e-06 warpx.abort_on_warning_threshold=medium dim = 3 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index bd3086c0438..3e3a1729736 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -88,7 +88,8 @@ WarpX::ComputeSpaceChargeField (bool const reset_fields) } // Add the field due to the boundary potentials - if (electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic){ + if (m_boundary_potential_specified || + (electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic)){ AddBoundaryField(); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index d3900c91593..6526171ff67 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -954,6 +954,7 @@ public: */ [[nodiscard]] amrex::IntVect get_numprocs() const {return numprocs;} + bool m_boundary_potential_specified = false; ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler; void ComputeSpaceChargeField (bool reset_fields); void AddBoundaryField (); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 6c3186c5c8d..c929718b043 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -765,13 +765,34 @@ WarpX::ReadParameters () } // Parse the input file for domain boundary potentials const ParmParse pp_boundary("boundary"); - pp_boundary.query("potential_lo_x", m_poisson_boundary_handler.potential_xlo_str); - pp_boundary.query("potential_hi_x", m_poisson_boundary_handler.potential_xhi_str); - pp_boundary.query("potential_lo_y", m_poisson_boundary_handler.potential_ylo_str); - pp_boundary.query("potential_hi_y", m_poisson_boundary_handler.potential_yhi_str); - pp_boundary.query("potential_lo_z", m_poisson_boundary_handler.potential_zlo_str); - pp_boundary.query("potential_hi_z", m_poisson_boundary_handler.potential_zhi_str); - pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); + bool potential_specified = false; + // When reading the potential at the boundary from the input file, set this flag to true if any of the potential is specified + potential_specified |= pp_boundary.query("potential_lo_x", m_poisson_boundary_handler.potential_xlo_str); + potential_specified |= pp_boundary.query("potential_hi_x", m_poisson_boundary_handler.potential_xhi_str); + potential_specified |= pp_boundary.query("potential_lo_y", m_poisson_boundary_handler.potential_ylo_str); + potential_specified |= pp_boundary.query("potential_hi_y", m_poisson_boundary_handler.potential_yhi_str); + potential_specified |= pp_boundary.query("potential_lo_z", m_poisson_boundary_handler.potential_zlo_str); + potential_specified |= pp_boundary.query("potential_hi_z", m_poisson_boundary_handler.potential_zhi_str); +#if defined(AMREX_USE_EB) + potential_specified |= pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); +#endif + m_boundary_potential_specified = potential_specified; + if (potential_specified & (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC)) { + ablastr::warn_manager::WMRecordWarning( + "Algorithms", + "The input script specifies the electric potential (phi) at the boundary, but \ + also uses the hybrid PIC solver based on Ohm’s law. When using this solver, the \ + electric potential does not have any impact on the simulation.", + ablastr::warn_manager::WarnPriority::low); + } + else if (potential_specified & (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None)) { + ablastr::warn_manager::WMRecordWarning( + "Algorithms", + "The input script specifies the electric potential (phi) at the boundary so \ + an initial Poisson solve will be performed.", + ablastr::warn_manager::WarnPriority::low); + } + m_poisson_boundary_handler.buildParsers(); #ifdef WARPX_DIM_RZ pp_boundary.query("verboncoeur_axis_correction", verboncoeur_axis_correction); From e145007861cdb7988fd99a095055ab38484b99ed Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 14 Mar 2024 12:31:11 -0700 Subject: [PATCH 008/121] Better error message when Poisson solver does not have supported boundary (#4778) * Better boundary error message * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Source/FieldSolver/ElectrostaticSolver.cpp | 8 ++++---- .../MagnetostaticSolver/MagnetostaticSolver.cpp | 12 ++++++------ Source/Utils/WarpXAlgorithmSelection.H | 5 +++++ Source/Utils/WarpXAlgorithmSelection.cpp | 12 ++++++++++++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 3e3a1729736..f1bda102c8b 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -1019,9 +1019,9 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo dirichlet_flag[idim*2] = false; } else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, + WARPX_ABORT_WITH_MESSAGE( "Field boundary conditions have to be either periodic, PEC or neumann " - "when using the electrostatic solver" + "when using the electrostatic solver, but they are " + GetFieldBCTypeString(WarpX::field_boundary_lo[idim]) ); } @@ -1034,9 +1034,9 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo dirichlet_flag[idim*2+1] = false; } else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, + WARPX_ABORT_WITH_MESSAGE( "Field boundary conditions have to be either periodic, PEC or neumann " - "when using the electrostatic solver" + "when using the electrostatic solver, but they are " + GetFieldBCTypeString(WarpX::field_boundary_hi[idim]) ); } } diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 192017656ce..26ac1ac96c8 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -318,9 +318,9 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs dirichlet_flag[adim][idim*2] = false; } else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary conditions have to be either periodic, PEC, or Neumann " - "when using the magnetostatic solver" + WARPX_ABORT_WITH_MESSAGE( + "Field boundary conditions have to be either periodic, PEC or neumann " + "when using the magnetostatic solver, but they are " + GetFieldBCTypeString(WarpX::field_boundary_lo[idim]) ); } @@ -338,9 +338,9 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs dirichlet_flag[adim][idim*2+1] = false; } else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary conditions have to be either periodic, PEC, or Neumann " - "when using the magnetostatic solver" + WARPX_ABORT_WITH_MESSAGE( + "Field boundary conditions have to be either periodic, PEC or neumann " + "when using the magnetostatic solver, but they are " + GetFieldBCTypeString(WarpX::field_boundary_lo[idim]) ); } } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 735fc7993f1..e94b7bb7719 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -188,4 +188,9 @@ GetFieldBCTypeInteger( std::string BCType ); ParticleBoundaryType GetParticleBCTypeInteger( std::string BCType ); +/** Find the name associated with a BC type + */ +std::string +GetFieldBCTypeString( int fb_type ); + #endif // UTILS_WARPXALGORITHMSELECTION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index abaf17f0a2c..b874c151f36 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -246,3 +246,15 @@ GetParticleBCTypeInteger( std::string BCType ){ // return ParticleBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map return ParticleBCType_algo_to_enum.at(BCType); } + +std::string +GetFieldBCTypeString( int fb_type ) { + std::string boundary_name = ""; + for (const auto &valid_pair : FieldBCType_algo_to_int) { + if ((valid_pair.second == fb_type)&&(valid_pair.first != "default")){ + boundary_name = valid_pair.first; + break; + } + } + return boundary_name; +} From 8c84e792db66ef8e2d99a4068daf837972d0a7aa Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 15 Mar 2024 09:57:21 -0700 Subject: [PATCH 009/121] CI: `isort` + `black` compatibility (#4769) * CI: `isort` + `black` compatibility Avoid that these tools follow slightly different rules. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove Unused Import Co-authored-by: Luca Fedeli --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Luca Fedeli --- .pre-commit-config.yaml | 1 + Docs/source/conf.py | 2 +- .../workflows/ml_materials/create_dataset.py | 2 +- .../ml_materials/run_warpx_training.py | 1 - .../usage/workflows/ml_materials/visualize.py | 2 +- .../capacitive_discharge/PICMI_inputs_1d.py | 3 +-- .../capacitive_discharge/PICMI_inputs_2d.py | 3 +-- .../Physics_applications/laser_ion/plot_2d.py | 4 ++-- .../spacecraft_charging/PICMI_inputs_rz.py | 3 +-- .../spacecraft_charging/analysis.py | 2 +- Examples/Tests/AcceleratorLattice/analysis.py | 2 +- Examples/Tests/Implicit/analysis_vandb_2d.py | 2 +- Examples/Tests/boosted_diags/analysis.py | 2 +- Examples/Tests/boundaries/analysis.py | 2 +- .../Tests/collision/analysis_collision_rz.py | 2 +- Examples/Tests/dive_cleaning/analysis.py | 2 +- .../analysis_electrostatic_sphere.py | 2 +- .../Tests/electrostatic_sphere_eb/analysis.py | 1 + .../electrostatic_sphere_eb/analysis_rz.py | 2 +- .../embedded_boundary_cube/analysis_fields.py | 2 +- .../analysis_fields_2d.py | 2 +- .../PICMI_inputs_EB_API.py | 1 - .../analysis_fields.py | 2 +- .../analysis_fields_2d.py | 2 +- .../analysis_flux_injection_3d.py | 2 +- .../analysis_distribution.py | 2 +- .../ion_stopping/analysis_ion_stopping.py | 2 +- Examples/Tests/langmuir/PICMI_inputs_rz.py | 1 - Examples/Tests/langmuir/analysis_2d.py | 2 +- Examples/Tests/langmuir/analysis_3d.py | 2 +- Examples/Tests/langmuir_fluids/analysis_2d.py | 2 +- Examples/Tests/langmuir_fluids/analysis_3d.py | 2 +- Examples/Tests/laser_injection/analysis_2d.py | 4 ++-- .../analysis_from_RZ_file.py | 5 +---- .../Tests/magnetostatic_eb/PICMI_inputs_3d.py | 1 - .../Tests/magnetostatic_eb/PICMI_inputs_rz.py | 1 - .../Tests/ohm_solver_EM_modes/PICMI_inputs.py | 3 +-- .../ohm_solver_EM_modes/PICMI_inputs_rz.py | 3 +-- .../Tests/ohm_solver_EM_modes/analysis.py | 1 - .../Tests/ohm_solver_EM_modes/analysis_rz.py | 7 +++---- .../PICMI_inputs.py | 3 +-- .../ohm_solver_ion_Landau_damping/analysis.py | 1 - .../PICMI_inputs.py | 3 +-- .../analysis.py | 1 - .../PICMI_inputs.py | 5 ++--- .../analysis.py | 2 +- .../PICMI_inputs_rz.py | 1 - .../particle_boundary_interaction/analysis.py | 2 +- .../PICMI_inputs_scrape.py | 1 - .../particle_data_python/PICMI_inputs_2d.py | 1 - .../PICMI_inputs_prev_pos_2d.py | 1 - .../analysis_particle_diags_impl.py | 2 +- .../pass_mpi_communicator/PICMI_inputs_2d.py | 1 - Examples/Tests/plasma_lens/analysis.py | 2 +- .../Tests/point_of_contact_EB/analysis.py | 2 +- .../Tests/python_wrappers/PICMI_inputs_2d.py | 3 +-- .../analysis_reduced_diags_impl.py | 2 +- .../repelling_particles/analysis_repelling.py | 2 +- .../resampling/analysis_leveling_thinning.py | 2 +- .../Tests/restart/PICMI_inputs_id_cpu_read.py | 1 - .../PICMI_inputs_runtime_component_analyze.py | 1 - .../analysis_rigid_injection_BoostedFrame.py | 1 - Examples/Tests/scraping/analysis_rz.py | 2 +- Examples/Tests/scraping/analysis_rz_filter.py | 2 +- .../space_charge_initialization/analysis.py | 2 +- Examples/Tests/vay_deposition/analysis.py | 2 +- Python/pywarpx/WarpX.py | 4 ++-- Python/pywarpx/__init__.py | 4 ++-- Python/pywarpx/particle_containers.py | 2 +- Python/pywarpx/picmi.py | 3 +-- Regression/Checksum/checksum.py | 4 ++-- Regression/prepare_file_ci.py | 1 + Tools/Algorithms/stencil.py | 2 +- Tools/LibEnsemble/run_libensemble_on_warpx.py | 19 +++++++++---------- Tools/LibEnsemble/warpx_simf.py | 2 +- Tools/PerformanceTests/run_automated.py | 4 ++-- .../plot_distribution_mapping.py | 2 +- Tools/PostProcessing/yt3d_mpi.py | 2 +- Tools/Release/updateAMReX.py | 2 +- Tools/Release/updatePICSAR.py | 2 +- Tools/Release/updatepyAMReX.py | 2 +- pyproject.toml | 3 +++ 82 files changed, 85 insertions(+), 110 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3d14c123c1..812a046b602 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,6 +80,7 @@ repos: hooks: - id: isort name: isort (python) + args: ['--profile black'] # Python: Flake8 (checks only, does this support auto-fixes?) #- repo: https://github.com/PyCQA/flake8 diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 1a439c8b76e..10752c21bda 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -31,8 +31,8 @@ import urllib.request import pybtex.plugin -from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme +from pybtex.style.formatting.unsrt import Style as UnsrtStyle sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py index 08000105479..1dd16f2f95e 100644 --- a/Docs/source/usage/workflows/ml_materials/create_dataset.py +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -15,8 +15,8 @@ c = 2.998e8 -from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker import torch +from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker ############### diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py index 8ee098e3e29..772fe574504 100644 --- a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -9,7 +9,6 @@ import math import numpy as np - from pywarpx import picmi # Physical constants diff --git a/Docs/source/usage/workflows/ml_materials/visualize.py b/Docs/source/usage/workflows/ml_materials/visualize.py index e6da1029b67..920efe29909 100644 --- a/Docs/source/usage/workflows/ml_materials/visualize.py +++ b/Docs/source/usage/workflows/ml_materials/visualize.py @@ -7,11 +7,11 @@ # Authors: Ryan Sandberg # License: BSD-3-Clause-LBNL # -from matplotlib import pyplot as plt import neural_network_classes as mynn import numpy as np import torch import torch.nn.functional as F +from matplotlib import pyplot as plt c = 2.998e8 diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 01406b7a0fb..d54800debd6 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -8,11 +8,10 @@ import sys import numpy as np +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla -from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi - constants = picmi.constants diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py index 65baabba605..61229a17295 100755 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py @@ -6,11 +6,10 @@ # --- used for the field solve step. import numpy as np +from pywarpx import callbacks, fields, picmi from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla -from pywarpx import callbacks, fields, picmi - constants = picmi.constants ########################## diff --git a/Examples/Physics_applications/laser_ion/plot_2d.py b/Examples/Physics_applications/laser_ion/plot_2d.py index e85782a7d23..736203e85ea 100644 --- a/Examples/Physics_applications/laser_ion/plot_2d.py +++ b/Examples/Physics_applications/laser_ion/plot_2d.py @@ -14,12 +14,12 @@ import os import re -from matplotlib.colors import TwoSlopeNorm import matplotlib.pyplot as plt import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import pandas as pd import scipy.constants as sc +from matplotlib.colors import TwoSlopeNorm +from openpmd_viewer import OpenPMDTimeSeries plt.rcParams.update({'font.size':16}) diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 08ebe23f828..3de56a93710 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -12,10 +12,9 @@ # --- of electrons - leads to a decrease of the potential on the surface over the time # --- until reaching an equilibrium floating potential of ~144.5 V (*). -from mpi4py import MPI as mpi import numpy as np import scipy.constants as scc - +from mpi4py import MPI as mpi from pywarpx import picmi from pywarpx.callbacks import installafterEsolve, installafterInitEsolve from pywarpx.fields import ExWrapper, EzWrapper, PhiFPWrapper, RhoFPWrapper diff --git a/Examples/Physics_applications/spacecraft_charging/analysis.py b/Examples/Physics_applications/spacecraft_charging/analysis.py index d8e8ac86af8..10786baab61 100755 --- a/Examples/Physics_applications/spacecraft_charging/analysis.py +++ b/Examples/Physics_applications/spacecraft_charging/analysis.py @@ -17,9 +17,9 @@ import matplotlib.pyplot as plt import numpy as np +import yt from openpmd_viewer import OpenPMDTimeSeries from scipy.optimize import curve_fit -import yt yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/AcceleratorLattice/analysis.py b/Examples/Tests/AcceleratorLattice/analysis.py index 55fb1b2ac96..5f3de4543d6 100755 --- a/Examples/Tests/AcceleratorLattice/analysis.py +++ b/Examples/Tests/AcceleratorLattice/analysis.py @@ -19,8 +19,8 @@ import sys import numpy as np -from scipy.constants import c, e, m_e import yt +from scipy.constants import c, e, m_e yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_2d.py index fa3299925a8..85faab61fcc 100755 --- a/Examples/Tests/Implicit/analysis_vandb_2d.py +++ b/Examples/Tests/Implicit/analysis_vandb_2d.py @@ -14,8 +14,8 @@ import sys import numpy as np -from scipy.constants import e, epsilon_0 import yt +from scipy.constants import e, epsilon_0 sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index c0b03f4a20b..2b21184dc3d 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -21,8 +21,8 @@ import numpy as np import openpmd_api as io -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/boundaries/analysis.py b/Examples/Tests/boundaries/analysis.py index 0a879bb83e5..9c108b16196 100755 --- a/Examples/Tests/boundaries/analysis.py +++ b/Examples/Tests/boundaries/analysis.py @@ -18,8 +18,8 @@ import sys import numpy as np -from scipy.constants import c, m_e import yt +from scipy.constants import c, m_e yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/collision/analysis_collision_rz.py b/Examples/Tests/collision/analysis_collision_rz.py index 9f9b9c17f5e..b206b2eba7b 100755 --- a/Examples/Tests/collision/analysis_collision_rz.py +++ b/Examples/Tests/collision/analysis_collision_rz.py @@ -16,9 +16,9 @@ # tolerance: 1.0e-30 # Possible running time: ~ 1.0 s -from glob import glob import os import sys +from glob import glob import numpy as np import yt diff --git a/Examples/Tests/dive_cleaning/analysis.py b/Examples/Tests/dive_cleaning/analysis.py index 504368fa85e..a9b52455baa 100755 --- a/Examples/Tests/dive_cleaning/analysis.py +++ b/Examples/Tests/dive_cleaning/analysis.py @@ -22,8 +22,8 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc -from scipy.special import gammainc import yt +from scipy.special import gammainc yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index 097ea0674f7..35281084c01 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -21,8 +21,8 @@ import sys import numpy as np -from scipy.optimize import fsolve import yt +from scipy.optimize import fsolve sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis.py b/Examples/Tests/electrostatic_sphere_eb/analysis.py index c1a5ba33a1b..bf2725616b5 100755 --- a/Examples/Tests/electrostatic_sphere_eb/analysis.py +++ b/Examples/Tests/electrostatic_sphere_eb/analysis.py @@ -9,6 +9,7 @@ sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI + # Check reduced diagnostics for charge on EB import numpy as np from scipy.constants import epsilon_0 diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py b/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py index 5d807a2dbd3..18aba4322f0 100755 --- a/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py @@ -20,8 +20,8 @@ import sys import numpy as np -from unyt import m import yt +from unyt import m sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_cube/analysis_fields.py b/Examples/Tests/embedded_boundary_cube/analysis_fields.py index 13bd32d73e4..dc6af9d57d2 100755 --- a/Examples/Tests/embedded_boundary_cube/analysis_fields.py +++ b/Examples/Tests/embedded_boundary_cube/analysis_fields.py @@ -5,8 +5,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py b/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py index 67e893b199a..8faa299025e 100755 --- a/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py +++ b/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py @@ -4,8 +4,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py index faec3ed4668..6104e08ef27 100755 --- a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py +++ b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import numpy as np - from pywarpx import fields, picmi max_steps = 1 diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py index 1f7b604365c..e849958468f 100755 --- a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py @@ -11,8 +11,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py index 5c8c544d0c4..dcdbc83a729 100755 --- a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py @@ -4,8 +4,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py index 470cbfe6065..d0271f6aa94 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py @@ -26,9 +26,9 @@ import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, m_e, m_p from scipy.special import erf -import yt sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/initial_distribution/analysis_distribution.py b/Examples/Tests/initial_distribution/analysis_distribution.py index d5301c077d2..5a3774133db 100755 --- a/Examples/Tests/initial_distribution/analysis_distribution.py +++ b/Examples/Tests/initial_distribution/analysis_distribution.py @@ -22,9 +22,9 @@ import sys import numpy as np -from read_raw_data import read_reduced_diags, read_reduced_diags_histogram import scipy.constants as scc import scipy.special as scs +from read_raw_data import read_reduced_diags, read_reduced_diags_histogram sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/ion_stopping/analysis_ion_stopping.py b/Examples/Tests/ion_stopping/analysis_ion_stopping.py index b090014c0cf..d7774c14d6b 100755 --- a/Examples/Tests/ion_stopping/analysis_ion_stopping.py +++ b/Examples/Tests/ion_stopping/analysis_ion_stopping.py @@ -16,8 +16,8 @@ import sys import numpy as np -from scipy.constants import e, epsilon_0, k, m_e, m_p import yt +from scipy.constants import e, epsilon_0, k, m_e, m_p sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 8328aba5185..106b8de5a94 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -9,7 +9,6 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np - from pywarpx import fields, picmi constants = picmi.constants diff --git a/Examples/Tests/langmuir/analysis_2d.py b/Examples/Tests/langmuir/analysis_2d.py index b3327703b82..94f97ca6de8 100755 --- a/Examples/Tests/langmuir/analysis_2d.py +++ b/Examples/Tests/langmuir/analysis_2d.py @@ -18,8 +18,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/langmuir/analysis_3d.py b/Examples/Tests/langmuir/analysis_3d.py index 2e56e64d641..68334f506ff 100755 --- a/Examples/Tests/langmuir/analysis_3d.py +++ b/Examples/Tests/langmuir/analysis_3d.py @@ -18,8 +18,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/langmuir_fluids/analysis_2d.py b/Examples/Tests/langmuir_fluids/analysis_2d.py index cf5d2fb44de..f7244f87137 100755 --- a/Examples/Tests/langmuir_fluids/analysis_2d.py +++ b/Examples/Tests/langmuir_fluids/analysis_2d.py @@ -17,8 +17,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/langmuir_fluids/analysis_3d.py b/Examples/Tests/langmuir_fluids/analysis_3d.py index 0211956859e..686907f103a 100755 --- a/Examples/Tests/langmuir_fluids/analysis_3d.py +++ b/Examples/Tests/langmuir_fluids/analysis_3d.py @@ -18,8 +18,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/laser_injection/analysis_2d.py b/Examples/Tests/laser_injection/analysis_2d.py index 37793171883..4424fe134bc 100755 --- a/Examples/Tests/laser_injection/analysis_2d.py +++ b/Examples/Tests/laser_injection/analysis_2d.py @@ -24,10 +24,10 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1 import make_axes_locatable import numpy as np -from scipy.signal import hilbert import yt +from mpl_toolkits.axes_grid1 import make_axes_locatable +from scipy.signal import hilbert sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py b/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py index a00d322bd18..44bbd00a7f4 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py +++ b/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py @@ -148,10 +148,7 @@ def launch_analysis(executable): def main() : from lasy.laser import Laser - from lasy.profiles import ( - CombinedLongitudinalTransverseProfile, - GaussianProfile, - ) + from lasy.profiles import CombinedLongitudinalTransverseProfile, GaussianProfile from lasy.profiles.longitudinal import GaussianLongitudinalProfile from lasy.profiles.transverse import LaguerreGaussianTransverseProfile diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py index 8f205724563..f61823bf223 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py @@ -27,7 +27,6 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as con - from pywarpx import fields, picmi ########################## diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 1268f7a02b0..6f7d6daa453 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -27,7 +27,6 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as con - from pywarpx import fields, picmi ########################## diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 4f3cd731323..5101e218129 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -12,9 +12,8 @@ import sys import dill -from mpi4py import MPI as mpi import numpy as np - +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 81ba65e5b28..e6f9ccc8e65 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -10,9 +10,8 @@ import sys import dill -from mpi4py import MPI as mpi import numpy as np - +from mpi4py import MPI as mpi from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis.py b/Examples/Tests/ohm_solver_EM_modes/analysis.py index ad832dc2f50..77be546937c 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis.py @@ -6,7 +6,6 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np - from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index 9004c24a05c..b96bc3b4629 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -3,16 +3,15 @@ # --- Analysis script for the hybrid-PIC example producing EM modes. import dill -from matplotlib import colors import matplotlib.pyplot as plt import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import scipy.fft as fft +from matplotlib import colors +from openpmd_viewer import OpenPMDTimeSeries +from pywarpx import picmi from scipy.interpolate import RegularGridInterpolator from scipy.special import j1, jn, jn_zeros -from pywarpx import picmi - constants = picmi.constants # load simulation parameters diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index d545195a9e5..8249bdb504f 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -11,9 +11,8 @@ import time import dill -from mpi4py import MPI as mpi import numpy as np - +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py index ade7187ebe3..93a4962d7e5 100755 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py @@ -6,7 +6,6 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np - from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index d9a71df6ac4..186ef6cede3 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -12,9 +12,8 @@ import time import dill -from mpi4py import MPI as mpi import numpy as np - +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index 6a57b1c1046..6388a4c4085 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -7,7 +7,6 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np - from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index a7f2e0da12d..5d4631e9c1e 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -8,14 +8,13 @@ # --- https://aip.scitation.org/doi/10.1063/1.4943893. import argparse -from pathlib import Path import shutil import sys +from pathlib import Path import dill -from mpi4py import MPI as mpi import numpy as np - +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py index e48d5c03fb2..0fb1c05ae1a 100755 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py @@ -5,9 +5,9 @@ import glob import dill -from matplotlib import colors import matplotlib.pyplot as plt import numpy as np +from matplotlib import colors plt.rcParams.update({'font.size': 20}) diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py index 4017a05d413..b9ea76e4a09 100644 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py @@ -4,7 +4,6 @@ # --- This input is a simple case of reflection # --- of one electron on the surface of a sphere. import numpy as np - from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/particle_boundary_interaction/analysis.py b/Examples/Tests/particle_boundary_interaction/analysis.py index 9768751ed74..ff83cc1fed7 100755 --- a/Examples/Tests/particle_boundary_interaction/analysis.py +++ b/Examples/Tests/particle_boundary_interaction/analysis.py @@ -11,8 +11,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py index e5a9a58f597..63a805c2612 100755 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py @@ -4,7 +4,6 @@ # --- to access the buffer of scraped particles. import numpy as np - from pywarpx import libwarpx, particle_containers, picmi ########################## diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index 572871b8ed5..f43c2215b79 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -3,7 +3,6 @@ import sys import numpy as np - from pywarpx import callbacks, libwarpx, particle_containers, picmi # Create the parser and add the argument diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index 5de9879f0f8..6f70b69fc10 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -3,7 +3,6 @@ # --- Input file to test the saving of old particle positions import numpy as np - from pywarpx import particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py b/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py index ceb71ffb4b7..5e54fc42d87 100755 --- a/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py +++ b/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py @@ -16,8 +16,8 @@ import numpy as np import openpmd_api as io -from scipy.constants import c, e, m_e, m_p import yt +from scipy.constants import c, e, m_e, m_p sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py index 66f259da2ef..095702a1145 100755 --- a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py +++ b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py @@ -6,7 +6,6 @@ # --- if the correct amount of processors are initialized in AMReX. from mpi4py import MPI - from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/plasma_lens/analysis.py b/Examples/Tests/plasma_lens/analysis.py index 9ce82d903f8..212e71087f9 100755 --- a/Examples/Tests/plasma_lens/analysis.py +++ b/Examples/Tests/plasma_lens/analysis.py @@ -19,8 +19,8 @@ import sys import numpy as np -from scipy.constants import c, e, m_e import yt +from scipy.constants import c, e, m_e yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py index cb1fc23ee62..042fc811a62 100755 --- a/Examples/Tests/point_of_contact_EB/analysis.py +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -11,8 +11,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py index 7104963c752..d01835f9fdf 100755 --- a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py +++ b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import numpy as np - +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable from pywarpx import picmi # Number of time steps diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py b/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py index 93352d29ff3..21359ed8171 100755 --- a/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py +++ b/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py @@ -15,11 +15,11 @@ import sys import numpy as np +import yt from scipy.constants import c from scipy.constants import epsilon_0 as eps0 from scipy.constants import m_e, m_p from scipy.constants import mu_0 as mu0 -import yt sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/repelling_particles/analysis_repelling.py b/Examples/Tests/repelling_particles/analysis_repelling.py index 0aa3ed53d1b..ebff010fc40 100755 --- a/Examples/Tests/repelling_particles/analysis_repelling.py +++ b/Examples/Tests/repelling_particles/analysis_repelling.py @@ -29,8 +29,8 @@ import matplotlib.pyplot as plt import numpy as np -from scipy.constants import c, m_e, physical_constants import yt +from scipy.constants import c, m_e, physical_constants yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/resampling/analysis_leveling_thinning.py b/Examples/Tests/resampling/analysis_leveling_thinning.py index 50fec628200..5f3dc8ecdff 100755 --- a/Examples/Tests/resampling/analysis_leveling_thinning.py +++ b/Examples/Tests/resampling/analysis_leveling_thinning.py @@ -13,8 +13,8 @@ import sys import numpy as np -from scipy.special import erf import yt +from scipy.special import erf sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py index d400924a378..d76cef82fec 100755 --- a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py +++ b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py @@ -6,7 +6,6 @@ import sys import numpy as np - from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index 32c9f4e5808..daac6e1b5ac 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -7,7 +7,6 @@ import sys import numpy as np - from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py b/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py index ccb55183241..fd51298816b 100755 --- a/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py +++ b/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py @@ -25,7 +25,6 @@ import numpy as np import openpmd_api as io -from scipy.constants import m_e import yt yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 7d40bd5edf4..193b2575992 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -24,8 +24,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py index dd13b993dc5..a4e0dafddbc 100755 --- a/Examples/Tests/scraping/analysis_rz_filter.py +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -23,8 +23,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries tolerance = 0 diff --git a/Examples/Tests/space_charge_initialization/analysis.py b/Examples/Tests/space_charge_initialization/analysis.py index 0df21e6d984..48e19ea0b75 100755 --- a/Examples/Tests/space_charge_initialization/analysis.py +++ b/Examples/Tests/space_charge_initialization/analysis.py @@ -20,8 +20,8 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc -from scipy.special import gammainc import yt +from scipy.special import gammainc yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/vay_deposition/analysis.py b/Examples/Tests/vay_deposition/analysis.py index f1680742ded..cfd089f2112 100755 --- a/Examples/Tests/vay_deposition/analysis.py +++ b/Examples/Tests/vay_deposition/analysis.py @@ -10,8 +10,8 @@ import sys import numpy as np -from scipy.constants import epsilon_0 import yt +from scipy.constants import epsilon_0 yt.funcs.mylog.setLevel(50) diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 3fa59285f26..6b24b35e918 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -9,6 +9,7 @@ import sys from . import Particles +from ._libwarpx import libwarpx from .Algo import algo from .Amr import amr from .Amrex import amrex @@ -22,9 +23,8 @@ from .HybridPICModel import hybridpicmodel from .Interpolation import interpolation from .Lasers import lasers, lasers_list -from .PSATD import psatd from .Particles import particles, particles_list -from ._libwarpx import libwarpx +from .PSATD import psatd class WarpX(Bucket): diff --git a/Python/pywarpx/__init__.py b/Python/pywarpx/__init__.py index d5181a0307c..037598f4ed4 100644 --- a/Python/pywarpx/__init__.py +++ b/Python/pywarpx/__init__.py @@ -23,6 +23,7 @@ if os.path.exists(p_abs): os.add_dll_directory(p_abs) +from ._libwarpx import libwarpx from .Algo import algo from .Amr import amr from .Amrex import amrex @@ -36,10 +37,9 @@ from .Interpolation import interpolation from .Lasers import lasers from .LoadThirdParty import load_cupy -from .PSATD import psatd from .Particles import newspecies, particles +from .PSATD import psatd from .WarpX import warpx -from ._libwarpx import libwarpx # This is a circular import and must happen after the import of libwarpx from . import picmi # isort:skip diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index f8e687f99bd..f5330d05f84 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -8,8 +8,8 @@ import numpy as np -from .LoadThirdParty import load_cupy from ._libwarpx import libwarpx +from .LoadThirdParty import load_cupy class ParticleContainerWrapper(object): diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 59ca1ec789f..911bbe13148 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -8,13 +8,12 @@ """Classes following the PICMI standard """ -from dataclasses import dataclass import os import re +from dataclasses import dataclass import numpy as np import periodictable - import picmistandard import pywarpx diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index 454edfc2606..727c8beb7f7 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -9,11 +9,11 @@ import json import sys -from benchmark import Benchmark import numpy as np +import yt +from benchmark import Benchmark from openpmd_viewer import OpenPMDTimeSeries from scipy.constants import c -import yt yt.funcs.mylog.setLevel(50) diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index d52d05b1139..ce7a3398b95 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -6,6 +6,7 @@ # License: BSD-3-Clause-LBNL import os + # This script modifies `WarpX-test.ini` (which is used for nightly builds) # and creates the file `ci-test.ini` (which is used for continuous # integration) diff --git a/Tools/Algorithms/stencil.py b/Tools/Algorithms/stencil.py index 63bb7f11c2e..dde7398daaa 100644 --- a/Tools/Algorithms/stencil.py +++ b/Tools/Algorithms/stencil.py @@ -16,9 +16,9 @@ import sys sys.path.append('../Parser/') -from input_file_parser import parse_input_file import matplotlib.pyplot as plt import numpy as np +from input_file_parser import parse_input_file from scipy.constants import c plt.style.use('tableau-colorblind10') diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py index b86c0249d01..d825288dc56 100644 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ b/Tools/LibEnsemble/run_libensemble_on_warpx.py @@ -21,20 +21,23 @@ import sys +import numpy as np + # Import libEnsemble modules from libensemble.libE import libE -import numpy as np from warpx_simf import run_warpx # Sim function from current directory if generator_type == 'random': - from libensemble.alloc_funcs.give_sim_work_first import \ - give_sim_work_first as alloc_f + from libensemble.alloc_funcs.give_sim_work_first import ( + give_sim_work_first as alloc_f, + ) from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f elif generator_type == 'aposmm': import libensemble.gen_funcs libensemble.gen_funcs.rc.aposmm_optimizers = 'nlopt' - from libensemble.alloc_funcs.persistent_aposmm_alloc import \ - persistent_aposmm_alloc as alloc_f + from libensemble.alloc_funcs.persistent_aposmm_alloc import ( + persistent_aposmm_alloc as alloc_f, + ) from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f else: print("you shouldn' hit that") @@ -43,11 +46,7 @@ import all_machine_specs from libensemble import libE_logger from libensemble.executors.mpi_executor import MPIExecutor -from libensemble.tools import ( - add_unique_random_streams, - parse_args, - save_libE_output, -) +from libensemble.tools import add_unique_random_streams, parse_args, save_libE_output # Import machine-specific run parameters if machine == 'local': diff --git a/Tools/LibEnsemble/warpx_simf.py b/Tools/LibEnsemble/warpx_simf.py index 4a1f0f90dee..f310dd0f95e 100644 --- a/Tools/LibEnsemble/warpx_simf.py +++ b/Tools/LibEnsemble/warpx_simf.py @@ -1,9 +1,9 @@ import os import time +import numpy as np from libensemble.executors.executor import Executor from libensemble.message_numbers import TASK_FAILED, WORKER_DONE -import numpy as np from read_sim_output import read_sim_output from write_sim_input import write_sim_input diff --git a/Tools/PerformanceTests/run_automated.py b/Tools/PerformanceTests/run_automated.py index 73ab79b00df..f03ead05376 100644 --- a/Tools/PerformanceTests/run_automated.py +++ b/Tools/PerformanceTests/run_automated.py @@ -13,14 +13,14 @@ import sys import time +import git +import pandas as pd from functions_perftest import ( extract_dataframe, get_file_content, run_batch_nnode, store_git_hash, ) -import git -import pandas as pd # Get name of supercomputer and import configuration functions from # machine-specific file diff --git a/Tools/PostProcessing/plot_distribution_mapping.py b/Tools/PostProcessing/plot_distribution_mapping.py index 19628551f26..4b0cdfd532b 100644 --- a/Tools/PostProcessing/plot_distribution_mapping.py +++ b/Tools/PostProcessing/plot_distribution_mapping.py @@ -1,7 +1,7 @@ # Standard imports +import os from collections import defaultdict from itertools import product -import os # High-performance math import numpy as np diff --git a/Tools/PostProcessing/yt3d_mpi.py b/Tools/PostProcessing/yt3d_mpi.py index 20054756d22..655327aff3d 100644 --- a/Tools/PostProcessing/yt3d_mpi.py +++ b/Tools/PostProcessing/yt3d_mpi.py @@ -19,10 +19,10 @@ import glob -from mpi4py import MPI import numpy as np import scipy.constants as scc import yt +from mpi4py import MPI yt.funcs.mylog.setLevel(50) diff --git a/Tools/Release/updateAMReX.py b/Tools/Release/updateAMReX.py index 4f2d84e4eb7..9dfa7fbeb41 100755 --- a/Tools/Release/updateAMReX.py +++ b/Tools/Release/updateAMReX.py @@ -9,9 +9,9 @@ # when building WarpX. # import datetime -from pathlib import Path import re import sys +from pathlib import Path import requests diff --git a/Tools/Release/updatePICSAR.py b/Tools/Release/updatePICSAR.py index ce1efec0a99..7e61679d371 100755 --- a/Tools/Release/updatePICSAR.py +++ b/Tools/Release/updatePICSAR.py @@ -9,9 +9,9 @@ # when building WarpX. # import datetime -from pathlib import Path import re import sys +from pathlib import Path import requests diff --git a/Tools/Release/updatepyAMReX.py b/Tools/Release/updatepyAMReX.py index 8ba10c895e0..500781e0880 100755 --- a/Tools/Release/updatepyAMReX.py +++ b/Tools/Release/updatepyAMReX.py @@ -9,9 +9,9 @@ # when building WarpX. # import datetime -from pathlib import Path import re import sys +from pathlib import Path import requests diff --git a/pyproject.toml b/pyproject.toml index f53522b5622..5173216a0f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,6 @@ requires = [ "packaging>=23", ] build-backend = "setuptools.build_meta" + +[tool.isort] +profile = "black" From 01ee64e4a6f9ab673db299859b2935d002e93dd8 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 15 Mar 2024 21:53:09 +0100 Subject: [PATCH 010/121] fix redundant string init found with clang-tidy (#4781) --- Source/Utils/WarpXAlgorithmSelection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index b874c151f36..c58318bbd53 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -249,7 +249,7 @@ GetParticleBCTypeInteger( std::string BCType ){ std::string GetFieldBCTypeString( int fb_type ) { - std::string boundary_name = ""; + std::string boundary_name; for (const auto &valid_pair : FieldBCType_algo_to_int) { if ((valid_pair.second == fb_type)&&(valid_pair.first != "default")){ boundary_name = valid_pair.first; From a8f48eaf76a7d467cb39cb5ae974317545a17192 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sat, 16 Mar 2024 00:26:18 +0100 Subject: [PATCH 011/121] Remove WarpX:: from ablastr (#4782) --- Source/ablastr/fields/VectorPoissonSolver.H | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 6e68d0fadb5..df2287df7f4 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -185,11 +185,10 @@ computeVectorPotential ( amrex::Vector > co relative_tolerance, absolute_tolerance ); // Synchronize the ghost cells, do halo exchange - ablastr::utils::communication::FillBoundary(*A[lev][adim], - A[lev][adim]->nGrowVect(), - WarpX::do_single_precision_comms, - geom[lev].periodicity(), - false); + ablastr::utils::communication::FillBoundary( + *A[lev][adim], A[lev][adim]->nGrowVect(), + do_single_precision_comms, + geom[lev].periodicity(), false); // needed for solving the levels by levels: // - coarser level is initial guess for finer level From 4e568adffc55c6f9b279098499e7db232aa2c540 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 15 Mar 2024 19:31:02 -0700 Subject: [PATCH 012/121] Define class function `defineAllParticleTiles` for `NamedComponentParticleContainer` (#4780) * Define function to define tiles, for a NamedComponentParticleContainer * Call parent class in WarpXParticleContainer --- Source/Particles/NamedComponentParticleContainer.H | 13 +++++++++++++ Source/Particles/WarpXParticleContainer.cpp | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index e7a7a20fad5..983d417d9c0 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -190,6 +190,19 @@ public: } } + void defineAllParticleTiles () noexcept + { + for (int lev = 0; lev <= amrex::ParticleContainerPureSoA::finestLevel(); ++lev) + { + for (auto mfi = amrex::ParticleContainerPureSoA::MakeMFIter(lev); mfi.isValid(); ++mfi) + { + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + amrex::ParticleContainerPureSoA::DefineAndReturnParticleTile(lev, grid_id, tile_id); + } + } + } + /** Return the name-to-index map for the compile-time and runtime-time real components */ [[nodiscard]] std::map getParticleComps () const noexcept { return particle_comps;} /** Return the name-to-index map for the compile-time and runtime-time integer components */ diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 6e6c0b27c85..912dd3a5a40 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1517,6 +1517,10 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) // without runtime component). void WarpXParticleContainer::defineAllParticleTiles () noexcept { + // Call the parent class's method + NamedComponentParticleContainer::defineAllParticleTiles(); + + // Resize the tmp_particle_data (no present in parent class) tmp_particle_data.resize(finestLevel()+1); for (int lev = 0; lev <= finestLevel(); ++lev) { @@ -1525,7 +1529,6 @@ void WarpXParticleContainer::defineAllParticleTiles () noexcept const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); tmp_particle_data[lev][std::make_pair(grid_id,tile_id)]; - DefineAndReturnParticleTile(lev, grid_id, tile_id); } } } From d22e423f6370e124dcaf61b7e3b2a4a36343df07 Mon Sep 17 00:00:00 2001 From: Ryan Sandberg Date: Sat, 16 Mar 2024 00:55:53 -0700 Subject: [PATCH 013/121] update ml workflow for pasc revisions (#4768) * update ml workflow for pasc revisions * update pasc reference: arxiv link+add andrew * update figures * Update Bibtex * refactor * clean duplication in references * add explicit reference to Zenodo archive --- Docs/source/acknowledge_us.rst | 5 ++ Docs/source/highlights.rst | 11 +++ Docs/source/refs.bib | 8 +- .../usage/workflows/ml_dataset_training.rst | 83 +++++++++++-------- .../workflows/ml_materials/create_dataset.py | 40 ++++----- .../ml_materials/run_warpx_training.py | 21 ++--- .../usage/workflows/ml_materials/train.py | 4 +- 7 files changed, 100 insertions(+), 72 deletions(-) diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index 79bb9fa8884..4c7a2d8137c 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -53,6 +53,11 @@ Prior WarpX references If your project uses a specific algorithm or component, please consider citing the respective publications in addition. +- Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint __` + - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 98b0d85391e..91c99ee50ca 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -24,6 +24,11 @@ Scientific works in laser-plasma and beam-plasma acceleration. Phys. Rev. Research **5**, 033112, 2023 `DOI:10.1103/PhysRevResearch.5.033112 `__ +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint __` + #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. @@ -96,6 +101,11 @@ Particle Accelerator & Beam Physics Scientific works in particle and beam modeling. +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint __` + #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. @@ -128,6 +138,7 @@ Microelectronics **Characterization of Transmission Lines in Microelectronic Circuits Using the ARTEMIS Solver**. IEEE Journal on Multiscale and Multiphysics Computational Techniques, vol. 8, pp. 31-39, 2023. `DOI:10.1109/JMMCT.2022.3228281 `__ + #. Kumar P, Nonaka A, Jambunathan R, Pahwa G and Salahuddin S, Yao Z. **FerroX: A GPU-accelerated, 3D Phase-Field Simulation Framework for Modeling Ferroelectric Devices**. arXiv preprint, 2022. diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 895a6c1392b..6c7665dac84 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -207,13 +207,15 @@ @article{Roedel2010 @misc{SandbergPASC24, address = {Zuerich, Switzerland}, -author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Andrew Myers and Ji Qiang and Jean-Luc Vay and Axel Huebl}, booktitle = {Proc. of PASC24}, -note = {submitted}, +note = {accepted}, series = {PASC'24 - Platform for Advanced Scientific Computing}, title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, venue = {Zuerich, Switzerland}, -year = {2024} +year = {2024}, +doi = {10.48550/arXiv.2402.17248}, +url = {https://arxiv.org/abs/2402.17248} } @inproceedings{SandbergIPAC23, diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst index 6e60a318bee..450e0fe2879 100644 --- a/Docs/source/usage/workflows/ml_dataset_training.rst +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -14,42 +14,42 @@ For example, a simulation determined by the following input script .. literalinclude:: ml_materials/run_warpx_training.py :language: python -In this section we walk through a workflow for data processing and model training. +In this section we walk through a workflow for data processing and model training, using data from this input script as an example. +The simulation output is stored in an online `Zenodo archive `__, in the ``lab_particle_diags`` directory. +In the example scripts provided here, the data is downloaded from the Zenodo archive, properly formatted, and used to train a neural network. This workflow was developed and first presented in :cite:t:`ml-SandbergIPAC23,ml-SandbergPASC24`. - -This assumes you have an up-to-date environment with PyTorch and openPMD. +It assumes you have an up-to-date environment with PyTorch and openPMD. Data Cleaning ------------- -It is important to inspect the data for artifacts to +It is important to inspect the data for artifacts, to check that input/output data make sense. -If we plot the final phase space for beams 1-8, -the particle data is distributed in a single blob, -as shown by :numref:`fig_phase_space_beam_1` for beam 1. -This is as we expect and what is optimal for training neural networks. +If we plot the final phase space of the particle beam, +shown in :numref:`fig_unclean_phase_space`. +we see outlying particles. +Looking closer at the z-pz space, we see that some particles were not trapped in the accelerating region of the wake and have much less energy than the rest of the beam. + +.. _fig_unclean_phase_space: -.. _fig_phase_space_beam_1: +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/unclean_stage_0.png + :alt: Plot showing the final phase space projections of a particle beam through a laser-plasma acceleration element where some beam particles were not accelerated. -.. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png - :alt: Plot showing the final phase space projections for beam 1 of the training data, for a surrogate to stage 1. + The final phase space projections of a particle beam through a laser-plasma acceleration element where some beam particles were not accelerated. - The final phase space projections for beam 1 of the training data, for a surrogate to stage 1. +To assist our neural network in learning dynamics of interest, we filter out these particles. +It is sufficient for our purposes to select particles that are not too far back, setting +``particle_selection={'z':[0.280025, None]}``. +After filtering, we can see in :numref:`fig_clean_phase_space` that the beam phase space projections are much cleaner -- this is the beam we want to train on. -.. _fig_phase_space_beam_0: +.. _fig_clean_phase_space: -.. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png - :alt: Plot showing the final phase space projections for beam 0 of the training data, for a surrogate to stage 0. +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/clean_stage_0.png + :alt: Plot showing the final phase space projections of a particle beam through a laser-plasma acceleration element after filtering out outlying particles. - The final phase space projections for beam 0 of the training data, for a surrogate to stage 0 + The final phase space projections of a particle beam through a laser-plasma acceleration element after filtering out outlying particles. -On the other hand, the final phase space for beam 0, shown in :numref:`fig_phase_space_beam_1`, -has a halo of outlying particles. -Looking closer at the z-pz space, we see that some particles got caught in a decelerating -region of the wake, have slipped back and are much slower than the rest of the beam. -To assist our neural network in learning dynamics of interest, we filter out these particles. -It is sufficient for our purposes to select particles that are not too far back, setting -``particle_selection={'z':[0.28002, None]}``. Then a particle tracker is set up to make sure +A particle tracker is set up to make sure we consistently filter out these particles from both the initial and final data. .. literalinclude:: ml_materials/create_dataset.py @@ -58,6 +58,9 @@ we consistently filter out these particles from both the initial and final data. :start-after: # Manual: Particle tracking START :end-before: # Manual: Particle tracking END +This data cleaning ensures that the particle data is distributed in a single blob, +as is optimal for training neural networks. + Create Normalized Dataset ------------------------- @@ -119,7 +122,12 @@ This data are converted to an :math:`N\times 6` numpy array and then to a PyTorc Save Normalizations and Normalized Data ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -With the data properly normalized, it and the normalizations are saved to file for +The data is split into training and testing subsets. +We take most of the data (70%) for training, meaning that data is used to update +the neural network parameters. +The testing data is reserved to determine how well the neural network generalizes; +that is, how well the neural network performs on data that wasn't used to update the neural network parameters. +With the data split and properly normalized, it and the normalizations are saved to file for use in training and inference. .. literalinclude:: ml_materials/create_dataset.py @@ -131,13 +139,22 @@ use in training and inference. Neural Network Structure ------------------------ -It was found in :cite:t:`ml-SandbergPASC24` that reasonable surrogate models are obtained with -shallow feedforward neural networks consisting of fewer than 10 hidden layers and -just under 1000 nodes per layer. +It was found in :cite:t:`ml-SandbergPASC24` that a reasonable surrogate model is obtained with +shallow feedforward neural networks consisting of about 5 hidden layers and 700-900 nodes per layer. The example shown here uses 3 hidden layers and 20 nodes per layer and is trained for 10 epochs. +Some utility functions for creating neural networks are provided in the script below. +These are mostly convenience wrappers and utilities for working with `PyTorch `__ neural network objects. +This script is imported in the training scripts shown later. + +.. dropdown:: Python neural network class definitions + :color: light + :icon: info + :animate: fade-in-slide-down + .. literalinclude:: ml_materials/neural_network_classes.py + :language: python3 Train and Save Neural Network ----------------------------- @@ -188,8 +205,8 @@ which is later divided by the size of the dataset in the training loop. :start-after: # Manual: Test function START :end-before: # Manual: Test function END -Train Loop -^^^^^^^^^^ +Training Loop +^^^^^^^^^^^^^ The full training loop performs ``n_epochs`` number of iterations. At each iteration the training and testing functions are called, @@ -228,14 +245,14 @@ When the test-loss starts to trend flat or even upward, the neural network is no .. _fig_train_test_loss: -.. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/beam_stage_0_training_testing_error.png :alt: Plot of training and testing loss curves versus number of training epochs. Training (in blue) and testing (in green) loss curves versus number of training epochs. .. _fig_train_evaluation: -.. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/beam_stage_0_model_evaluation.png :alt: Plot comparing model prediction with simulation output. A comparison of model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). @@ -243,7 +260,7 @@ When the test-loss starts to trend flat or even upward, the neural network is no A visual inspection of the model prediction can be seen in :numref:`fig_train_evaluation`. This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. -A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, +A more accurate model is obtained with 5 hidden layers and 900 nodes per layer, as discussed in :cite:t:`ml-SandbergPASC24`. These figures can be generated with the following Python script. @@ -261,7 +278,7 @@ Surrogate Usage in Accelerator Physics ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A neural network such as the one we trained here can be incorporated in other BLAST codes. -`Consider the example using neural networks in ImpactX `__. +Consider this `example using neural network surrogates of WarpX simulations in ImpactX `__. .. bibliography:: :keyprefix: ml- diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py index 1dd16f2f95e..ecd182c8802 100644 --- a/Docs/source/usage/workflows/ml_materials/create_dataset.py +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -7,17 +7,16 @@ # Authors: Ryan Sandberg # License: BSD-3-Clause-LBNL # + import os -import tarfile +import zipfile from urllib import request import numpy as np - -c = 2.998e8 - import torch from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker +c = 2.998e8 ############### def sanitize_dir_strings(*dir_strings): @@ -31,14 +30,15 @@ def sanitize_dir_strings(*dir_strings): return dir_strings def download_and_unzip(url, data_dir): - request.urlretrieve(url, data_dir) - with tarfile.open(data_dir) as tar_dataset: - tar_dataset.extractall() + request.urlretrieve(url, data_dir) + with zipfile.ZipFile(data_dir, 'r') as zip_dataset: + zip_dataset.extractall() def create_source_target_data(data_dir, species, source_index=0, target_index=-1, + survivor_select_index=-1, particle_selection=None ): """Create dataset from openPMD files @@ -67,7 +67,7 @@ def create_source_target_data(data_dir, relevant_times = [ts.t[source_index], ts.t[target_index]] # Manual: Particle tracking START - iteration = ts.iterations[target_index] + iteration = ts.iterations[survivor_select_index] pt = ParticleTracker( ts, species=species, iteration=iteration, @@ -116,7 +116,6 @@ def create_source_target_data(data_dir, return source_data, source_means, source_stds, target_data, target_means, target_stds, relevant_times - def save_warpx_surrogate_data(dataset_fullpath_filename, diag_dir, species, @@ -128,12 +127,14 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, particle_selection=None ): - source_target_data = create_source_target_data(data_dir=diag_dir, - species=species, - source_index=source_index, - target_index=target_index, - particle_selection=particle_selection - ) + source_target_data = create_source_target_data( + data_dir=diag_dir, + species=species, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + particle_selection=particle_selection + ) source_data, source_means, source_stds, target_data, target_means, target_stds, times = source_target_data # Manual: Save dataset START @@ -161,11 +162,10 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, ######## end utility functions ############# ######## start dataset creation ############ -data_url = "https://zenodo.org/records/10368972/files/ml_example_training.tar.gz?download=1" -download_and_unzip(data_url, "training_dataset.tar.gz") +data_url = "https://zenodo.org/records/10810754/files/lab_particle_diags.zip?download=1" +download_and_unzip(data_url, "lab_particle_diags.zip") data_dir = "lab_particle_diags/lab_particle_diags/" - # create data set source_index = 0 @@ -178,7 +178,7 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, # improve stage 0 dataset stage_i = 0 -select = {'z':[0.28002, None]} +select = {'z':[0.280025, None]} species = f'beam_stage_{stage_i}' dataset_filename = f'dataset_{species}.pt' dataset_file = 'datasets/' + dataset_filename @@ -193,7 +193,7 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, particle_selection=select ) -for stage_i in range(1,9): +for stage_i in range(1,15): species = f'beam_stage_{stage_i}' dataset_filename = f'dataset_{species}.pt' dataset_file = 'datasets/' + dataset_filename diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py index 772fe574504..23fc49e7431 100644 --- a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -1,11 +1,4 @@ #!/usr/bin/env python3 -# -# Copyright 2022-2023 WarpX contributors -# Authors: WarpX team -# License: BSD-3-Clause-LBNL -# -# -*- coding: utf-8 -*- - import math import numpy as np @@ -21,7 +14,7 @@ # Number of cells dim = '3' nx = ny = 128 -nz = 8832 +nz = 35328 #17664 #8832 if dim == 'rz': nr = nx//2 @@ -33,9 +26,9 @@ # Number of processes for static load balancing # Check with your submit script -num_procs = [1, 1, 16*4] +num_procs = [1, 1, 64*4] if dim == 'rz': - num_procs = [1, 16] + num_procs = [1, 64] # Number of time steps gamma_boost = 60. @@ -73,7 +66,7 @@ # plasma region plasma_rlim = 100.e-6 -N_stage = 9 +N_stage = 15 L_plasma_bulk = 0.28 L_ramp = 1.e-9 L_ramp_up = L_ramp @@ -141,12 +134,12 @@ def get_species_of_accelerator_stage(stage_idx, stage_zmin, stage_zmax, N_beam_particles = int(1e6) beam_centroid_z = -107.e-6 beam_rms_z = 2.e-6 -#beam_gammas = [2000 + 13000 * i_stage for i_stage in range(N_stage)] -beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run +beam_gammas = [1960 + 13246 * i_stage for i_stage in range(N_stage)] +#beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run beams = [] for i_stage in range(N_stage): beam_gamma = beam_gammas[i_stage] - sigma_gamma = 0.10 * beam_gamma + sigma_gamma = 0.06 * beam_gamma gaussian_distribution = picmi.GaussianBunchDistribution( n_physical_particles= abs(beam_charge) / q_e, rms_bunch_size=[2.e-6, 2.e-6, beam_rms_z], diff --git a/Docs/source/usage/workflows/ml_materials/train.py b/Docs/source/usage/workflows/ml_materials/train.py index dc62819061c..4de11b9c99e 100644 --- a/Docs/source/usage/workflows/ml_materials/train.py +++ b/Docs/source/usage/workflows/ml_materials/train.py @@ -20,8 +20,8 @@ stage_i = 0 species = f'beam_stage_{stage_i}' source_index = 0 -target_index = 4 -survivor_select_index = 4 +target_index = 1 +survivor_select_index = 1 data_dim = 6 n_in = data_dim From 6610e4731dd6ceb261e0ac41766d5d2528dcf793 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 18 Mar 2024 18:54:13 -0700 Subject: [PATCH 014/121] Add callbacks after B and E field pushes (#4784) --- Python/pywarpx/callbacks.py | 18 ++++++++++++++++++ Source/FieldSolver/WarpXPushFieldsEM.cpp | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index 8f3c1dc5e2c..1f32fee6620 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -30,6 +30,8 @@ * ``beforeEsolve``: before the solve for E fields * ``poissonsolver``: In place of the computePhi call but only in an electrostatic simulation * ``afterEsolve``: after the solve for E fields +* ``afterBpush``: after the B field advance for electromagnetic solvers +* ``afterEpush``: after the E field advance for electromagnetic solvers * ``beforedeposition``: before the particle deposition (for charge and/or current) * ``afterdeposition``: after particle deposition (for charge and/or current) * ``beforestep``: before the time step @@ -276,6 +278,8 @@ def callfuncsinlist(self,*args,**kw): "beforeEsolve": {}, "poissonsolver": {'singlefunconly': True}, # external Poisson solver "afterEsolve": {}, + "afterBpush": {}, + "afterEpush": {}, "beforedeposition": {}, "afterdeposition": {}, "particlescraper": {}, @@ -401,6 +405,20 @@ def callfromafterEsolve(f): def installafterEsolve(f): installcallback('afterEsolve', f) +# ---------------------------------------------------------------------------- +def callfromafterBpush(f): + installcallback('afterBpush', f) + return f +def installafterBpush(f): + installcallback('afterBpush', f) + +# ---------------------------------------------------------------------------- +def callfromafterEpush(f): + installcallback('afterEpush', f) + return f +def installafterEpush(f): + installcallback('afterEpush', f) + # ---------------------------------------------------------------------------- def callfrombeforedeposition(f): installcallback('beforedeposition', f) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index bb9ec04f266..fe7642fd172 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -20,6 +20,7 @@ # include "FieldSolver/SpectralSolver/SpectralSolver.H" # endif #endif +#include "Python/callbacks.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -798,6 +799,9 @@ WarpX::EvolveB (amrex::Real a_dt, DtType a_dt_type) for (int lev = 0; lev <= finest_level; ++lev) { EvolveB(lev, a_dt, a_dt_type); } + + // Allow execution of Python callback after B-field push + ExecutePythonCallback("afterBpush"); } void @@ -848,6 +852,9 @@ WarpX::EvolveE (amrex::Real a_dt) { EvolveE(lev, a_dt); } + + // Allow execution of Python callback after E-field push + ExecutePythonCallback("afterEpush"); } void From 0531910462a324b3369c9fee9206f4e6634b003e Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 19 Mar 2024 16:54:47 +0100 Subject: [PATCH 015/121] Add UB sanitizer CI test (#4783) * add UB sanitizer CI test * fix bug * use fno-sanitize-recover=all * fix bug * rename sanitizers -> UB sanitizer * fix issue found with UB sanitizer * change compilation options * return early if np <= 0 --- .github/workflows/ubuntu.yml | 55 +++++++++++++++++++ .../FieldProbeParticleContainer.cpp | 8 ++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index cf4b375ce00..dcb975aaa66 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -218,6 +218,61 @@ jobs: export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + build_UB_sanitizer: + name: Clang UB sanitizer + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang15.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-15) + export CC=$(which clang-15) + export CXXFLAGS="-fsanitize=undefined -fno-sanitize-recover=all" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_PSATD=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE \ + -DWarpX_PARTICLE_PRECISION=SINGLE + cmake --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with UB sanitizer + run: | + export OMP_NUM_THREADS=2 + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d + save_pr_number: if: github.event_name == 'pull_request' runs-on: ubuntu-latest diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 7e7aecb9167..dc26de3c1e6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -76,6 +76,10 @@ FieldProbeParticleContainer::AddNParticles (int lev, // number of particles to add auto const np = static_cast(x.size()); + if (np <= 0){ + Redistribute(); + return; + } // have to resize here, not in the constructor because grids have not // been built when constructor was called. @@ -122,8 +126,8 @@ FieldProbeParticleContainer::AddNParticles (int lev, pinned_tile.push_back_real(FieldProbePIdx::Bz, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::S, np, 0.0); - auto old_np = particle_tile.numParticles(); - auto new_np = old_np + pinned_tile.numParticles(); + const auto old_np = particle_tile.numParticles(); + const auto new_np = old_np + pinned_tile.numParticles(); particle_tile.resize(new_np); amrex::copyParticles( particle_tile, pinned_tile, 0, old_np, pinned_tile.numParticles()); From 9b282be0691640ec264a3aa0ccfce22b9eaf8511 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:36:39 -0700 Subject: [PATCH 016/121] ApplyPECto Rho and J if partBndry reflecting/FieldBndry PEC (#4785) * ApplyPECto Rho and J if particle boundary is reflecting * rename functions * call reflecting J/rho if particleBoundary is reflecting/field boundary is PEC * semicolon * semicolong * fix comments --- .../WarpXFieldBoundaries.cpp | 8 +++- Source/BoundaryConditions/WarpX_PEC.H | 22 +++++---- Source/BoundaryConditions/WarpX_PEC.cpp | 45 ++++++++++++------- 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index af7a385b729..a9e2cf29fe9 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -106,14 +106,18 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoRhofield(rho, lev, patch_type); } + if (PEC::isAnyParticleBoundaryReflecting() || PEC::isAnyBoundaryPEC()) { + PEC::ApplyReflectiveBoundarytoRhofield(rho, lev, patch_type); + } } void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } + if (PEC::isAnyParticleBoundaryReflecting() || PEC::isAnyBoundaryPEC()) { + PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, lev, patch_type); + } } #ifdef WARPX_DIM_RZ diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index 62b92ba94b5..b0d15a70321 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -355,12 +355,13 @@ using namespace amrex; /** - * \brief Sets the rho or J field value in cells close to and on a PEC boundary. The - * charge/current density deposited in the guard cells are either reflected + * \brief Sets the rho or J field value in cells close to and on reflecting particle boundary + * or PEC field boundary. The charge/current density deposited + * in the guard cells are either reflected * back into the simulation domain (if a reflecting particle * boundary is used), or the opposite charge/current density is deposited * back in the domain to capture the effect of an image charge. - * The charge/current density on the PEC boundary is set to 0 while values + * The charge/current density on the reflecting boundary is set to 0 while values * in the guard cells are set equal (and opposite) to their mirror * location inside the domain - representing image charges - in the * normal (tangential) direction. @@ -370,7 +371,7 @@ using namespace amrex; * \param[in out] field field data to be updated * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_pec Whether the given boundary is PEC + * \param[in] is_reflective Whether the given particle boundary is reflecting or field boundary is pec * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary * \param[in] fabbox multifab box including ghost cells */ @@ -380,7 +381,7 @@ using namespace amrex; amrex::Array4 const& field, amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, amrex::GpuArray, AMREX_SPACEDIM> const& psign, - amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, + amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, amrex::GpuArray const& tangent_to_bndy, amrex::Box const& fabbox) { @@ -391,7 +392,7 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) { continue; } + if (!is_reflective[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; @@ -412,7 +413,7 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) { continue; } + if (!is_reflective[idim][iside]) { continue; } amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; @@ -479,6 +480,9 @@ using namespace amrex; /** Returns 1 if any domain boundary is set to PEC, else returns 0.*/ bool isAnyBoundaryPEC(); + + /** Return true if any particle boundary is set to reflecting, else returns false*/ + bool isAnyParticleBoundaryReflecting (); /** * \brief Sets the tangential electric field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell @@ -513,7 +517,7 @@ using namespace amrex; * \param[in] lev level of the Multifab * \param[in] patch_type coarse or fine */ - void ApplyPECtoRhofield(amrex::MultiFab* rho, int lev, + void ApplyReflectiveBoundarytoRhofield(amrex::MultiFab* rho, int lev, PatchType patch_type); /** @@ -524,7 +528,7 @@ using namespace amrex; * \param[in] lev level of the Multifab * \param[in] patch_type coarse or fine */ - void ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, + void ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, int lev, PatchType patch_type); diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 7555e2aaef2..0b4b4dae30a 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -24,6 +24,15 @@ PEC::isAnyBoundaryPEC() { return false; } +bool +PEC::isAnyParticleBoundaryReflecting () { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) {return true;} + if (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) {return true;} + } + return false; +} + void PEC::ApplyPECtoEfield (std::array Efield, const int lev, PatchType patch_type, const bool split_pml_field) @@ -222,7 +231,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, * location inside the domain - representing image charges. **/ void -PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_type) +PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_type) { auto& warpx = WarpX::GetInstance(); @@ -243,15 +252,17 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // cells for boundaries that are NOT PEC amrex::Box grown_domain_box = domain_box; - amrex::GpuArray, AMREX_SPACEDIM> is_pec; + amrex::GpuArray, AMREX_SPACEDIM> is_reflective; amrex::GpuArray is_tangent_to_bndy; amrex::GpuArray, AMREX_SPACEDIM> psign; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } - if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } + is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); + if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } // rho values inside guard cells are updated the same as tangential // components of the current density @@ -293,7 +304,7 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); PEC::SetRhoOrJfieldFromPEC( - n, iv, rho_array, mirrorfac, psign, is_pec, + n, iv, rho_array, mirrorfac, psign, is_reflective, is_tangent_to_bndy, fabbox ); }); @@ -302,7 +313,7 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty void -PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, +PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, const int lev, PatchType patch_type) { @@ -335,15 +346,17 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // directions of the current density multifab const amrex::IntVect ng_fieldgather = Jx->nGrowVect(); - amrex::GpuArray, AMREX_SPACEDIM> is_pec; + amrex::GpuArray, AMREX_SPACEDIM> is_reflective; amrex::GpuArray, 3> is_tangent_to_bndy; amrex::GpuArray, AMREX_SPACEDIM>, 3> psign; amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } - if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } + is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); + if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } for (int icomp=0; icomp < 3; ++icomp) { // Set the psign value correctly for each current component for each @@ -415,7 +428,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); PEC::SetRhoOrJfieldFromPEC( - n, iv, Jx_array, mirrorfac[0], psign[0], is_pec, + n, iv, Jx_array, mirrorfac[0], psign[0], is_reflective, is_tangent_to_bndy[0], fabbox ); }); @@ -450,7 +463,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); PEC::SetRhoOrJfieldFromPEC( - n, iv, Jy_array, mirrorfac[1], psign[1], is_pec, + n, iv, Jy_array, mirrorfac[1], psign[1], is_reflective, is_tangent_to_bndy[1], fabbox ); }); @@ -485,7 +498,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); PEC::SetRhoOrJfieldFromPEC( - n, iv, Jz_array, mirrorfac[2], psign[2], is_pec, + n, iv, Jz_array, mirrorfac[2], psign[2], is_reflective, is_tangent_to_bndy[2], fabbox ); }); From 2c61bc83358ac0fad44a86b3f944c8a0df3682b6 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 19 Mar 2024 19:46:31 -0700 Subject: [PATCH 017/121] Allow to set the initial potential in PICMI EM simulations (#4788) --- Python/pywarpx/picmi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 911bbe13148..dfebd13eeb3 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1680,7 +1680,6 @@ def embedded_boundary_initialize_inputs(self, solver): pywarpx.eb2.cover_multiple_cuts = self.cover_multiple_cuts if self.potential is not None: - assert isinstance(solver, ElectrostaticSolver), Exception('The potential is only supported with the ElectrostaticSolver') expression = pywarpx.my_constants.mangle_expression(self.potential, self.mangle_dict) pywarpx.warpx.__setattr__('eb_potential(x,y,z,t)', expression) From 29a0a564cd662a6bfb47b63510c232821d651b2a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 20 Mar 2024 06:01:07 +0100 Subject: [PATCH 018/121] Add QED lookup tables generator/reader in /Tools (#3137) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * initial commit * added support for BW table * table generator is working * added reader * fixed cmake issue * add test * fixed bug * fixed bug * add table generation to test * add terminal newline * added diff test * increase precision of table reader * fixed bug in table reading * fixed bug * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * correct test * added documentation * complete test * modified CI test * fixed issues * fix bug * Update .github/workflows/ubuntu.yml Co-authored-by: Axel Huebl * Update Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst Co-authored-by: Axel Huebl * fixed bug * add entry in cmake.rst * using script * fixed bug * change permission of script * removed 'false' from script * add newline at the end of a file * added new logic to control OpenMP support for lookup table generation * document cmake parameter * fixed trailing space * Frontier/Crusher: rocFFT Cache Control (#3366) rocFFT in ROCm 5.1+ tries to [write to a cache](https://rocfft.readthedocs.io/en/latest/library.html#runtime-compilation) in the home area by default. This does not scale. * `SyncCurrent`: Split Filter and Sum over Guard Cells (#3222) * Split ApplyFilterandSumBoundaryJ * Fix Bug * Fix Bug * Fix CI * Pass Correct Periodicity to SumBoundaryJ * Use ApplyFilterJ and SumBoundaryJ w/ MR * Fix Bug (duplicate loops over dimensions) * Use ApplyFilterJ and SumBoundaryJ w/ MR * Clean up Variable Names in ApplyFilterJ * Increase Guard Cells of All MultiFabs, Not Only J * Reset CI Checksums * Update Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> * Update Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> * remove wrong comment * improve documentation * updated test * fix bug * use the right executable to run test simulation * FIX BUG * Build & Test Cleanups --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> --- .../scripts/checkQEDTableGenerator.sh | 77 ++++++ .github/workflows/ubuntu.yml | 10 +- CMakeLists.txt | 20 +- Docs/source/install/cmake.rst | 2 + Docs/source/usage/parameters.rst | 4 + Docs/source/usage/workflows.rst | 1 + .../generate_lookup_tables_with_tools.rst | 45 ++++ ...erAcceleration_single_precision_comms.json | 2 +- Tools/QedTablesUtils/CMakeLists.txt | 35 +++ .../Source/ArgParser/QedTablesArgParser.H | 58 +++++ .../Source/ArgParser/QedTablesArgParser.cpp | 108 ++++++++ Tools/QedTablesUtils/Source/QedTableCommons.H | 26 ++ .../Source/QedTableGenerator.cpp | 239 +++++++++++++++++ .../QedTablesUtils/Source/QedTableReader.cpp | 240 ++++++++++++++++++ cmake/WarpXFunctions.cmake | 1 + cmake/dependencies/PICSAR.cmake | 25 +- 16 files changed, 879 insertions(+), 14 deletions(-) create mode 100755 .github/workflows/scripts/checkQEDTableGenerator.sh create mode 100644 Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst create mode 100644 Tools/QedTablesUtils/CMakeLists.txt create mode 100644 Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H create mode 100644 Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp create mode 100644 Tools/QedTablesUtils/Source/QedTableCommons.H create mode 100644 Tools/QedTablesUtils/Source/QedTableGenerator.cpp create mode 100644 Tools/QedTablesUtils/Source/QedTableReader.cpp diff --git a/.github/workflows/scripts/checkQEDTableGenerator.sh b/.github/workflows/scripts/checkQEDTableGenerator.sh new file mode 100755 index 00000000000..e14a7a2d6f2 --- /dev/null +++ b/.github/workflows/scripts/checkQEDTableGenerator.sh @@ -0,0 +1,77 @@ + +#!/usr/bin/env bash +# +# Copyright 2022 Luca Fedeli +# +# License: BSD-3-Clause-LBNL + +set -eu -o pipefail + +export OMP_NUM_THREADS=2 + +# +# Generate QED lookup tables using external tool +# +./build/bin/qed_table_generator \ + --table BW --mode DP --dndt_chi_min 0.01 \ + --dndt_chi_max 100 --dndt_how_many 64 \ + --pair_chi_min 0.01 --pair_chi_max 100 \ + --pair_chi_how_many 64 --pair_frac_how_many 64 \ + -o bw_table_tool +./build/bin/qed_table_generator \ + --table QS --mode DP --dndt_chi_min 0.001 \ + --dndt_chi_max 100 --dndt_how_many 64 \ + --em_chi_min 0.001 --em_chi_max 100 \ + --em_chi_how_many 64 --em_frac_how_many 64 \ + --em_frac_min 1e-12 -o qs_table_tool + +# +# Generate QED lookup tables using WarpX +# +./build/bin/warpx.2d \ + ./Examples/Tests/qed/quantum_synchrotron/inputs_2d \ + qed_bw.lookup_table_mode = "generate" \ + qed_bw.tab_dndt_chi_min = 0.01 \ + qed_bw.tab_dndt_chi_max = 100.0 \ + qed_bw.tab_dndt_how_many = 64 \ + qed_bw.tab_pair_chi_min = 0.01 \ + qed_bw.tab_pair_chi_max = 100.0 \ + qed_bw.tab_pair_chi_how_many = 64 \ + qed_bw.tab_pair_frac_how_many = 64 \ + qed_bw.save_table_in = "bw_table" \ + qed_qs.lookup_table_mode = "generate" \ + qed_qs.tab_dndt_chi_min = 0.001 \ + qed_qs.tab_dndt_chi_max = 100.0 \ + qed_qs.tab_dndt_how_many = 64 \ + qed_qs.tab_em_chi_min = 0.001 \ + qed_qs.tab_em_frac_min = 1.0e-12 \ + qed_qs.tab_em_chi_max = 100.0 \ + qed_qs.tab_em_chi_how_many = 64 \ + qed_qs.tab_em_frac_how_many = 64 \ + qed_qs.save_table_in = "qs_table" + +# +# Convert lookup tables (generated with WarpX and with the external tool) in human-readable format +# +./build/bin/qed_table_reader -i qs_table --table QS --mode DP -o qs_table +./build/bin/qed_table_reader -i qs_table_tool --table QS --mode DP -o qs_table_tool +./build/bin/qed_table_reader -i bw_table --table BW --mode DP -o bw_table +./build/bin/qed_table_reader -i bw_table_tool --table BW --mode DP -o bw_table_tool + +# +# Compare the generated lookup tables +# +diff bw_table_dndt bw_table_tool_dndt +diff bw_table_pair bw_table_tool_pair +diff qs_table_phot_em qs_table_tool_phot_em +diff qs_table_dndt qs_table_tool_dndt + +# +# Run a WarpX simulation using the lookup tables generated by the external tool +# +./build/bin/warpx.2d \ + ./Examples/Tests/qed/quantum_synchrotron/inputs_2d \ + qed_bw.lookup_table_mode = "load" \ + qed_bw.load_table_from = "bw_table_tool" \ + qed_qs.lookup_table_mode = "load" \ + qed_qs.load_table_from = "qs_table_tool" diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index dcb975aaa66..2997e9cdd16 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -46,7 +46,7 @@ jobs: du -hs ~/.cache/ccache build_1D_2D: - name: GCC 1D & 2D w/ MPI + name: GCC 1D & 2D w/ MPI, QED tools runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false env: @@ -78,7 +78,9 @@ jobs: -DWarpX_DIMS="1;2" \ -DWarpX_EB=OFF \ -DWarpX_PSATD=ON \ - -DWarpX_QED_TABLE_GEN=ON + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_QED_TOOLS=ON + cmake --build build -j 4 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d @@ -86,6 +88,10 @@ jobs: ccache -s du -hs ~/.cache/ccache + - name: run QED table tools + run: | + .github/workflows/scripts/checkQEDTableGenerator.sh + build_3D_sp: name: GCC 3D & RZ w/ MPI, single precision runs-on: ubuntu-22.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ddc685c733..e1d5dca6e3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,15 +71,19 @@ option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) option(WarpX_EB "Embedded boundary support" OFF) cmake_dependent_option(WarpX_GPUCLOCK "Add GPU kernel timers (cost function)" ON - "WarpX_COMPUTE STREQUAL CUDA OR WarpX_COMPUTE STREQUAL HIP" OFF) + "WarpX_COMPUTE STREQUAL CUDA OR WarpX_COMPUTE STREQUAL HIP" + OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_PSATD "spectral solver support" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) -option(WarpX_QED "QED support (requires PICSAR)" ON) -option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) +option(WarpX_QED "QED support (requires PICSAR)" ON) +option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" + OFF) +option(WarpX_QED_TOOLS "Build external tool to generate QED lookup tables (requires PICSAR and Boost)" + OFF) set(WarpX_DIMS_VALUES 1 2 3 RZ) set(WarpX_DIMS 3 CACHE STRING "Simulation dimensionality <1;2;3;RZ>") @@ -111,6 +115,13 @@ if(NOT WarpX_PARTICLE_PRECISION IN_LIST WarpX_PARTICLE_PRECISION_VALUES) message(FATAL_ERROR "WarpX_PARTICLE_PRECISION (${WarpX_PARTICLE_PRECISION}) must be one of ${WarpX_PARTICLE_PRECISION_VALUES}") endif() +set(WarpX_QED_TABLES_GEN_OMP_VALUES AUTO ON OFF) +set(WarpX_QED_TABLES_GEN_OMP AUTO CACHE STRING "Enables OpenMP support for QED lookup tables generation (AUTO/ON/OFF)") +set_property(CACHE WarpX_QED_TABLES_GEN_OMP PROPERTY STRINGS ${WarpX_QED_TABLES_GEN_OMP_VALUES}) +if(NOT WarpX_QED_TABLES_GEN_OMP IN_LIST WarpX_QED_TABLES_GEN_OMP_VALUES) + message(FATAL_ERROR "WarpX_QED_TABLES_GEN_OMP (${WarpX_QED_TABLES_GEN_OMP}) must be one of ${WarpX_QED_TABLES_GEN_OMP_VALUES}") +endif() + set(WarpX_COMPUTE_VALUES NOACC OMP CUDA SYCL HIP) set(WarpX_COMPUTE OMP CACHE STRING "On-node, accelerated computing backend (NOACC/OMP/CUDA/SYCL/HIP)") set_property(CACHE WarpX_COMPUTE PROPERTY STRINGS ${WarpX_COMPUTE_VALUES}) @@ -402,6 +413,9 @@ if(WarpX_LIB) add_subdirectory(Source/Python) add_subdirectory(Source/Utils) endif() +if(WarpX_QED_TOOLS) + add_subdirectory(Tools/QedTablesUtils) +endif() # Interprocedural optimization (IPO) / Link-Time Optimization (LTO) if(WarpX_IPO) diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 4474b3509fe..7ff8986cf36 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -101,6 +101,8 @@ CMake Option Default & Values Descr ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) +``WarpX_QED_TOOLS`` ON/**OFF** Build external tool to generate QED lookup tables (requires PICSAR and Boost) +``WarpX_QED_TABLES_GEN_OMP`` **AUTO**/ON/OFF Enables OpenMP support for QED lookup tables generation ``WarpX_SENSEI`` ON/**OFF** SENSEI in situ visualization ============================= ============================================ ========================================================= diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a615c9c8a87..bf66c24ccb9 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -3319,6 +3319,8 @@ Lookup tables store pre-computed values for functions used by the QED modules. * ``qed_bw.save_table_in`` (`string`): where to save the lookup table + Alternatively, the lookup table can be generated using a standalone tool (see :ref:`qed tools section `). + * ``load``: a lookup table is loaded from a pre-generated binary file. The following parameter must be specified: @@ -3356,6 +3358,8 @@ Lookup tables store pre-computed values for functions used by the QED modules. * ``qed_qs.save_table_in`` (`string`): where to save the lookup table + Alternatively, the lookup table can be generated using a standalone tool (see :ref:`qed tools section `). + * ``load``: a lookup table is loaded from a pre-generated binary file. The following parameter must be specified: diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index e68ec9391be..b68810a4345 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -12,6 +12,7 @@ This section collects typical user workflows and best practices for WarpX. workflows/domain_decomposition workflows/plot_distribution_mapping workflows/debugging + workflows/generate_lookup_tables_with_tools.rst workflows/libensemble workflows/plot_timestep_duration workflows/psatd_stencil diff --git a/Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst b/Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst new file mode 100644 index 00000000000..7417b7a1f96 --- /dev/null +++ b/Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst @@ -0,0 +1,45 @@ +.. _generate-lookup-tables-with-tools: + +Generate QED lookup tables using the standalone tool +==================================================== + +We provide tools to generate and convert into a human-readable format the QED lookup tables. +Such tools can be compiled with ``cmake`` by setting the flag ``WarpX_QED_TOOLS=ON`` (this +requires both ``PICSAR`` and ``Boost`` libraries). The tools are compiled alongside the WarpX executable +in the folder ``bin``. We report here the help message displayed by the tools: + +.. code-block:: console + + $ ./qed_table_reader -h + ### QED Table Reader ### + Command line options: + -h [NO ARG] Prints all command line arguments + -i [STRING] Name of the file to open + --table [STRING] Either BW (Breit-Wheeler) or QS (Quantum Synchrotron) + --mode [STRING] Precision of the calculations: either DP (double) or SP (single) + -o [STRING] filename to save the lookup table in human-readable format + + $ ./qed_table_generator -h + ### QED Table Generator ### + Command line options: + -h [NO ARG] Prints all command line arguments + --table [STRING] Either BW (Breit-Wheeler) or QS (Quantum Synchrotron) + --mode [STRING] Precision of the calculations: either DP (double) or SP (single) + --dndt_chi_min [DOUBLE] minimum chi parameter for the dNdt table + --dndt_chi_max [DOUBLE] maximum chi parameter for the dNdt table + --dndt_how_many [INTEGR] number of points in the dNdt table + --pair_chi_min [DOUBLE] minimum chi for the pair production table (BW only) + --pair_chi_max [DOUBLE] maximum chi for the pair production table (BW only) + --pair_chi_how_many [INTEGR] number of chi points in the pair production table (BW only) + --pair_frac_how_many [INTEGR] number of frac points in the pair production table (BW only) + --em_chi_min [DOUBLE] minimum chi for the photon emission table (QS only) + --em_chi_max [DOUBLE] maximum chi for the photon emission production table (QS only) + --em_frac_min [DOUBLE] minimum frac for the photon emission production table (QS only) + --em_chi_how_many [INTEGR] number of chi points in the photon emission table (QS only) + --em_frac_how_many [INTEGR] number of frac points in the photon emission table (QS only) + -o [STRING] filename to save the lookup table + +These tools are meant to be compatible with WarpX: ``qed_table_generator`` should generate +tables that can be loaded into WarpX and ``qed_table_reader`` should be able to read tables generated with WarpX. +It is not safe to use these tools to generate a table on a machine using a different endianness with respect to +the machine where the table is used. diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json index 30eef0cedb7..90493698df0 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json @@ -22,4 +22,4 @@ "particle_momentum_z": 4.231730764749506e-20, "particle_weight": 12926557617.187498 } -} \ No newline at end of file +} diff --git a/Tools/QedTablesUtils/CMakeLists.txt b/Tools/QedTablesUtils/CMakeLists.txt new file mode 100644 index 00000000000..09399316096 --- /dev/null +++ b/Tools/QedTablesUtils/CMakeLists.txt @@ -0,0 +1,35 @@ +# Common argument parser functions ############################################ +# +# TODO: Move to ABLASTR +set(arg_parser_src + Source/ArgParser/QedTablesArgParser.cpp + Source/ArgParser/QedTablesArgParser.H +) + + +# Build QED lookup tables generator ########################################### +# +add_executable(qed_table_generator + Source/QedTableGenerator.cpp + ${arg_parser_src} +) +add_executable(WarpX::qed_table_generator ALIAS qed_table_generator) + +target_link_libraries(qed_table_generator PRIVATE PXRMP_QED) + +target_compile_features(qed_table_generator PUBLIC cxx_std_17) +set_target_properties(qed_table_generator PROPERTIES CXX_EXTENSIONS OFF) + + +# Build QED lookup tables reader ############################################## +# +add_executable(qed_table_reader + Source/QedTableReader.cpp + ${arg_parser_src} +) +add_executable(WarpX::qed_table_reader ALIAS qed_table_reader) + +target_link_libraries(qed_table_reader PRIVATE PXRMP_QED) + +target_compile_features(qed_table_reader PUBLIC cxx_std_17) +set_target_properties(qed_table_reader PROPERTIES CXX_EXTENSIONS OFF) diff --git a/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H new file mode 100644 index 00000000000..f05b84ddca8 --- /dev/null +++ b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H @@ -0,0 +1,58 @@ +#ifndef QED_TABLES_ARG_PARSER_ +#define QED_TABLES_ARG_PARSER_ + +#include +#include +#include +#include +#include +#include +#include + +namespace ArgParser +{ + /* Possible types of parameters of command line arguments (no argument, int, double, or std::string) */ + enum class ArgType { NoArg, Integer, Double, String }; + + /* This type represents the type of a command line argument: if it exists it is either an int, a double, or a std::string */ + using ArgVal = std::optional>; + + /* The type of a possible command line argument */ + using Key = std::tuple; + + /* The type of the map of the parsed command line arguments */ + using ParsedArgs = std::unordered_map; + + /** + * \brief Gets the value out of an ArgVal (std::optional>) object + * + * \tparam T the type to return (must be int, double, or std::string) + * \param[in] arg_val the ArgVal object + * \return the value in arg_val + */ + template + T GetVal(const ArgVal& arg_val) + { + return std::get(*arg_val); + } + + /** + * \brief Function to parse the command line arguments + * + * \param[in] keys the list of possible command line arguments + * \param[in] argc the number of command line arguments + * \param[in] argv all the command line arguments + * \return the parsed command line arguments + */ + ParsedArgs ParseArgs (const std::vector& keys, const int argc, char const* const* argv); + + /** + * \brief Prints the command line options + * + * \param[in] cmd_list the list of possible command line arguments + */ + void PrintHelp (const std::vector& cmd_list); +}; + + +#endif //QED_TABLES_ARG_PARSER_ diff --git a/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp new file mode 100644 index 00000000000..41d27477dcf --- /dev/null +++ b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp @@ -0,0 +1,108 @@ +#include "QedTablesArgParser.H" + +#include + +using namespace ArgParser; +using namespace std; + +namespace +{ +void +WarnMsg(const string& msg) +{ + cout << "!!! Parse Warning: " + msg + "\n"; +} + +void +ErrMsg(const string& msg) +{ + cout << "### Parse Error: " + msg + "\n"; + exit(1); +} + +vector +ArgvToStrVec(const int argc, char const* const* argv ) +{ + auto sargs = vector{}; + for (int i = 0; i < argc; ++i) + sargs.push_back(std::string{argv[i]}); + return sargs; +} + +optional> +GetArgType(const std::vector& keys, vector::iterator& it) +{ + const auto str = *it; + it++; + for (const auto& kk : keys){ + if (get<0>(kk) == str) + { + return std::make_pair(get<0>(kk), get<1>(kk)); + } + } + return nullopt; +} + +ArgVal +ReadArg(const ArgType arg_type, vector::iterator& it) +{ + if(arg_type == ArgType::NoArg) return nullopt; + if(arg_type == ArgType::String) return *(it++); + if(arg_type == ArgType::Integer) return stoi(*(it++)); + if(arg_type == ArgType::Double) return stod(*(it++)); + + ErrMsg("Failed to parse type!"s); + + return nullopt; +} + +} + +ParsedArgs +ArgParser::ParseArgs (const std::vector& keys, const int argc, char const* const* argv) +{ + auto parsed_args = ParsedArgs{}; + + auto sargs = ArgvToStrVec(argc, argv); + auto it = sargs.begin() + 1; // the first argument is always the executable name + while (it < sargs.end()){ + const auto tt = *it; + auto res = GetArgType(keys, it); + if (res != nullopt){ + const auto& [key, arg_type] = *res; + const auto arg = ReadArg(arg_type, it); + if (parsed_args.find(key) != parsed_args.end()) + WarnMsg("Rewriting '"+ key +"' argument !"); + parsed_args[key] = arg; + }else{ + WarnMsg("Can't parse '"+ tt +"' !"); + } + } + + return parsed_args; +} + +void +ArgParser::PrintHelp (const vector& cmd_list) +{ + cout << "Command line options: " << endl; + + for (const auto& el : cmd_list){ + const auto type = get<1>(el); + string stype = "[??????]"; + if (type == ArgType::NoArg) + stype = "[NO ARG]"; + else if (type == ArgType::String) + stype = "[STRING]"; + else if (type == ArgType::Double) + stype = "[DOUBLE]"; + else if (type == ArgType::Integer) + stype = "[INTEGR]"; + + cout << get<0>(el) + << " " << stype + << " " << get<2>(el) + << endl; + } + +} diff --git a/Tools/QedTablesUtils/Source/QedTableCommons.H b/Tools/QedTablesUtils/Source/QedTableCommons.H new file mode 100644 index 00000000000..40551b9e13c --- /dev/null +++ b/Tools/QedTablesUtils/Source/QedTableCommons.H @@ -0,0 +1,26 @@ +#ifndef QED_TABLES_COMMONS_ +#define QED_TABLES_COMMONS_ + +#include +#include + +template +bool Contains (const ContainerType& container, const ElementType& el) +{ + return container.find(el) != std::end(container); +} + +void AbortWithMessage(const std::string& msg) +{ + std::cout << "### ABORT : " << msg << std::endl; + std::cout << "___________________________" << std::endl; + exit(1); +} + +void SuccessExit() +{ + std::cout << "___________________________" << std::endl; + exit(0); +} + +#endif //QED_TABLES_COMMONS_ diff --git a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp new file mode 100644 index 00000000000..dfbde1df050 --- /dev/null +++ b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp @@ -0,0 +1,239 @@ +#include "QedTableCommons.H" +#include "ArgParser/QedTablesArgParser.H" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace ArgParser; +using namespace std; + +namespace pxr_sr = picsar::multi_physics::utils::serialization; +namespace pxr_bw = picsar::multi_physics::phys::breit_wheeler; +namespace pxr_qs = picsar::multi_physics::phys::quantum_sync; + +const auto line_commands = vector{ + {"-h" , ArgType::NoArg , "Prints all command line arguments"}, + {"--table" , ArgType::String , "Either BW (Breit-Wheeler) or QS (Quantum Synchrotron)"}, + {"--mode" , ArgType::String, "Precision of the calculations: either DP (double) or SP (single)"}, + {"--dndt_chi_min" , ArgType::Double , "minimum chi parameter for the dNdt table"}, + {"--dndt_chi_max" , ArgType::Double , "maximum chi parameter for the dNdt table"}, + {"--dndt_how_many" , ArgType::Integer, "number of points in the dNdt table"}, + {"--pair_chi_min" , ArgType::Double , "minimum chi for the pair production table (BW only)"}, + {"--pair_chi_max" , ArgType::Double , "maximum chi for the pair production table (BW only)"}, + {"--pair_chi_how_many" , ArgType::Integer, "number of chi points in the pair production table (BW only)"}, + {"--pair_frac_how_many", ArgType::Integer, "number of frac points in the pair production table (BW only)"}, + {"--em_chi_min" , ArgType::Double , "minimum chi for the photon emission table (QS only)"}, + {"--em_chi_max" , ArgType::Double , "maximum chi for the photon emission production table (QS only)"}, + {"--em_frac_min" , ArgType::Double , "minimum frac for the photon emission production table (QS only)"}, + {"--em_chi_how_many" , ArgType::Integer, "number of chi points in the photon emission table (QS only)"}, + {"--em_frac_how_many" , ArgType::Integer, "number of frac points in the photon emission table (QS only)"}, + {"-o" , ArgType::String , "filename to save the lookup table"} +}; + +void GenerateTable (const ParsedArgs& args); +template +void GenerateTableBW (const ParsedArgs& args, const string& outfile_name); +template +void GenerateTableQS (const ParsedArgs& args, const string& outfile_name); + +int main (int argc, char** argv) +{ + cout << "### QED Table Generator ###" << endl; + const auto args_map = ParseArgs(line_commands, argc, argv); + + if (args_map.empty() || Contains(args_map, "-h")){ + PrintHelp(line_commands); + } + else{ + GenerateTable(args_map); + } + + SuccessExit(); +} + + +void GenerateTable (const ParsedArgs& args) +{ + if (!Contains(args, "--table")) + AbortWithMessage("'--table' argument must be provided"); + + if (!Contains(args, "--mode")) + AbortWithMessage("'--mode' argument must be provided"); + + if (!Contains(args, "-o")) + AbortWithMessage("'-o' argument must be provided"); + + const auto which_table = GetVal(args.at("--table")); + const auto mode = GetVal(args.at("--mode")); + const auto outfile_name = GetVal(args.at("-o")); + + bool use_double = false; + + if (mode == "DP"s) + use_double = true; + else if (mode == "SP"s) + use_double = false; + else + AbortWithMessage("'--mode' must be eiter 'DP' or 'SP'"); + + if (which_table == "BW"s){ + if (use_double) + GenerateTableBW(args, outfile_name); + else + GenerateTableBW(args, outfile_name); + } + else if (which_table == "QS"s) + if (use_double) + GenerateTableQS(args, outfile_name); + else + GenerateTableQS(args, outfile_name); + else + AbortWithMessage("'--table' must be eiter 'QS' or 'BW'"); +} + +template +void GenerateTableBW (const ParsedArgs& args, const string& outfile_name) +{ + cout << " Generating BW table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || + !Contains(args, "--dndt_how_many") || !Contains(args, "--pair_chi_min") || + !Contains(args, "--pair_chi_max") || !Contains(args, "--pair_chi_how_many") || + !Contains(args, "--pair_frac_how_many")) + AbortWithMessage("All the BW table arguments must be provided (check with -h)"); + + const auto dndt_table_params = + pxr_bw::dndt_lookup_table_params{ + static_cast(GetVal(args.at("--dndt_chi_min"))), + static_cast(GetVal(args.at("--dndt_chi_max"))), + GetVal(args.at("--dndt_how_many")) + }; + + const auto pair_prod_table_params = + pxr_bw::pair_prod_lookup_table_params{ + static_cast(GetVal(args.at("--pair_chi_min"))), + static_cast(GetVal(args.at("--pair_chi_max"))), + GetVal(args.at("--pair_chi_how_many")), + GetVal(args.at("--pair_frac_how_many")) + }; + + std::cout << " Params: \n"; + std::cout << " - dndt_chi_min : " << dndt_table_params.chi_phot_min << "\n"; + std::cout << " - dndt_chi_max : " << dndt_table_params.chi_phot_max << "\n"; + std::cout << " - dndt_how_many : " << dndt_table_params.chi_phot_how_many << "\n"; + std::cout << " - pair_chi_min : " << pair_prod_table_params.chi_phot_min << "\n"; + std::cout << " - pair_chi_max : " << pair_prod_table_params.chi_phot_max << "\n"; + std::cout << " - pair_chi_how_many : " << pair_prod_table_params.chi_phot_how_many << "\n"; + std::cout << " - pair_frac_how_many : " << pair_prod_table_params.frac_how_many << "\n"; + std::cout << " ----------------------- " << "\n\n"; + + auto dndt_table = + pxr_bw::dndt_lookup_table>{ + dndt_table_params}; + + auto pair_prod_table = + pxr_bw::pair_prod_lookup_table>{ + pair_prod_table_params}; + + dndt_table.generate(true); //Progress bar is displayed + pair_prod_table.generate(true); //Progress bar is displayed + + const auto data_dndt = dndt_table.serialize(); + const auto data_pair_prod = pair_prod_table.serialize(); + + const uint64_t size_first = data_dndt.size(); + + vector res{}; + pxr_sr::put_in(size_first, res); + for (const auto& tmp : data_dndt) + pxr_sr::put_in(tmp, res); + for (const auto& tmp : data_pair_prod) + pxr_sr::put_in(tmp, res); + + auto of = std::ofstream{outfile_name, std::ios_base::binary}; + of.write(res.data(), res.size()); + of.close(); + + cout << " Done! \n"; +} + +template +void GenerateTableQS (const ParsedArgs& args, const string& outfile_name) +{ + cout << " Generating QS table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || + !Contains(args, "--dndt_how_many") || !Contains(args, "--em_chi_min") || + !Contains(args, "--em_chi_max") || !Contains(args, "--em_frac_min") || + !Contains(args, "--em_chi_how_many") || !Contains(args, "--em_frac_how_many")) + AbortWithMessage("All the QS table arguments must be provided (check with -h)"); + + const auto dndt_table_params = + pxr_qs::dndt_lookup_table_params{ + static_cast(GetVal(args.at("--dndt_chi_min"))), + static_cast(GetVal(args.at("--dndt_chi_max"))), + GetVal(args.at("--dndt_how_many")) + }; + + const auto phot_em_table_params = + pxr_qs::photon_emission_lookup_table_params{ + static_cast(GetVal(args.at("--em_chi_min"))), + static_cast(GetVal(args.at("--em_chi_max"))), + static_cast(GetVal(args.at("--em_frac_min"))), + GetVal(args.at("--em_chi_how_many")), + GetVal(args.at("--em_frac_how_many")) + }; + + std::cout << " Params: \n"; + std::cout << " - dndt_chi_min : " << dndt_table_params.chi_part_min << "\n"; + std::cout << " - dndt_chi_max : " << dndt_table_params.chi_part_max << "\n"; + std::cout << " - dndt_how_many : " << dndt_table_params.chi_part_how_many << "\n"; + std::cout << " - phot_em_chi_min : " << phot_em_table_params.chi_part_min << "\n"; + std::cout << " - phot_em_chi_max : " << phot_em_table_params.chi_part_max << "\n"; + std::cout << " - phot_em_frac_min : " << phot_em_table_params.frac_min << "\n"; + std::cout << " - phot_em_chi_how_many : " << phot_em_table_params.chi_part_how_many << "\n"; + std::cout << " - phot_em_frac_how_many : " << phot_em_table_params.frac_how_many << "\n"; + std::cout << " ----------------------- " << "\n\n"; + + auto dndt_table = + pxr_qs::dndt_lookup_table>{ + dndt_table_params}; + + auto phot_em_table = + pxr_qs::photon_emission_lookup_table>{ + phot_em_table_params}; + + dndt_table.generate(true); //Progress bar is displayed + phot_em_table.generate(true); //Progress bar is displayed + + const auto data_dndt = dndt_table.serialize(); + const auto data_phot_em = phot_em_table.serialize(); + + const uint64_t size_first = data_dndt.size(); + + vector res{}; + pxr_sr::put_in(size_first, res); + for (const auto& tmp : data_dndt) + pxr_sr::put_in(tmp, res); + for (const auto& tmp : data_phot_em) + pxr_sr::put_in(tmp, res); + + auto of = std::ofstream{outfile_name, std::ios_base::binary}; + of.write(res.data(), res.size()); + of.close(); + + cout << " Done! \n"; +} diff --git a/Tools/QedTablesUtils/Source/QedTableReader.cpp b/Tools/QedTablesUtils/Source/QedTableReader.cpp new file mode 100644 index 00000000000..ae689a87643 --- /dev/null +++ b/Tools/QedTablesUtils/Source/QedTableReader.cpp @@ -0,0 +1,240 @@ +#include "QedTableCommons.H" +#include "ArgParser/QedTablesArgParser.H" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ArgParser; +using namespace std; + +namespace pxr_sr = picsar::multi_physics::utils::serialization; +namespace pxr_bw = picsar::multi_physics::phys::breit_wheeler; +namespace pxr_qs = picsar::multi_physics::phys::quantum_sync; + +const auto line_commands = vector{ + {"-h" , ArgType::NoArg , "Prints all command line arguments"}, + {"-i" , ArgType::String , "Name of the file to open"}, + {"--table" , ArgType::String , "Either BW (Breit-Wheeler) or QS (Quantum Synchrotron)"}, + {"--mode" , ArgType::String , "Precision of the calculations: either DP (double) or SP (single)"}, + {"-o" , ArgType::String , "filename to save the lookup table in human-readable format"} +}; + +void ReadTable (const ParsedArgs& args); +template +void ReadTableBW (const string& input_file, const string& outfile_name); +template +void ReadTableQS ( + const string& input_file, const string& outfile_name); + +/*Wrapper class to access protected data*/ +template +class bw_pair_production_table_wrapper : + public pxr_bw::pair_prod_lookup_table> +{ + public: + void write_table_data(std::ofstream& of) const; +}; + +/*Wrapper class to access protected data*/ +template +class qs_photon_emission_table_wrapper : + public pxr_qs::photon_emission_lookup_table> +{ + public: + void write_table_data(std::ofstream& of) const; +}; + +int main (int argc, char** argv) +{ + cout << "### QED Table Reader ###" << endl; + const auto args_map = ParseArgs(line_commands, argc, argv); + + if (args_map.empty() || Contains(args_map, "-h")){ + PrintHelp(line_commands); + } + else{ + ReadTable(args_map); + } + + SuccessExit(); +} + +void ReadTable (const ParsedArgs& args) +{ + if (!Contains(args, "--table")) + AbortWithMessage("'--table' argument must be provided"); + + if (!Contains(args, "--mode")) + AbortWithMessage("'--mode' argument must be provided"); + + if (!Contains(args, "-o")) + AbortWithMessage("'-o' argument must be provided"); + + if (!Contains(args, "-i")) + AbortWithMessage("'-i' argument must be provided"); + + const auto which_table = GetVal(args.at("--table")); + const auto mode = GetVal(args.at("--mode")); + const auto input_file = GetVal(args.at("-i")); + const auto outfile_name = GetVal(args.at("-o")); + + bool use_double = false; + + if (mode == "DP"s) + use_double = true; + else if (mode == "SP"s) + use_double = false; + else + AbortWithMessage("'--mode' must be eiter 'DP' or 'SP'"); + + if (which_table == "BW"s){ + if (use_double) + ReadTableBW(input_file, outfile_name); + else + ReadTableBW(input_file, outfile_name); + } + else if (which_table == "QS"s) + if (use_double) + ReadTableQS(input_file, outfile_name); + else + ReadTableQS(input_file, outfile_name); + else + AbortWithMessage("'--table' must be eiter 'QS' or 'BW'"); +} + +template +void ReadTableBW (const string& input_file, const string& outfile_name) +{ + cout << " Reading BW table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + auto ifs = ifstream(input_file, std::ios::binary); + auto raw_data = vector(istreambuf_iterator(ifs), {}); + ifs.close(); + + auto raw_iter = raw_data.begin(); + const auto size_first = pxr_sr::get_out(raw_iter); + if(size_first <= 0 || size_first >= raw_data.size() ) + AbortWithMessage("Something is wrong with " + input_file); + + const auto raw_dndt_table = vector{ + raw_iter, raw_iter+size_first}; + + const auto raw_pair_prod_table = vector{ + raw_iter+size_first, raw_data.end()}; + + const auto dndt_table = + pxr_bw::dndt_lookup_table>{raw_dndt_table}; + const auto pair_prod_table = + pxr_bw::pair_prod_lookup_table>{raw_pair_prod_table}; + + if (!dndt_table.is_init() || !pair_prod_table.is_init()) + AbortWithMessage("Something went wrong with lookup table initialization"); + + auto of_dndt = ofstream{outfile_name + "_dndt"}; + of_dndt << std::setprecision(std::numeric_limits::digits10 + 1); + const auto coord_dndt = dndt_table.get_all_coordinates(); + for (const auto& cc : coord_dndt ) + of_dndt << cc << " " << dndt_table.interp(cc) << "\n"; + of_dndt.close(); + + auto of_pair = ofstream{outfile_name + "_pair"}; + of_pair << std::setprecision(std::numeric_limits::digits10 + 1); + const auto wrapper = bw_pair_production_table_wrapper{pair_prod_table}; + wrapper.write_table_data(of_pair); + of_pair.close(); +} + +template +void ReadTableQS ( + const string& input_file, const string& outfile_name) +{ + cout << " Reading QS table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + auto ifs = ifstream(input_file, std::ios::binary); + auto raw_data = vector(istreambuf_iterator(ifs), {}); + ifs.close(); + + auto raw_iter = raw_data.begin(); + const auto size_first = pxr_sr::get_out(raw_iter); + if(size_first <= 0 || size_first >= raw_data.size() ) + AbortWithMessage("Something is wrong with " + input_file); + + const auto raw_dndt_table = vector{ + raw_iter, raw_iter+size_first}; + + const auto raw_phot_em_table = vector{ + raw_iter+size_first, raw_data.end()}; + + const auto dndt_table = + pxr_qs::dndt_lookup_table>{raw_dndt_table}; + const auto phot_em_table = + pxr_qs::photon_emission_lookup_table>{raw_phot_em_table}; + + if (!dndt_table.is_init() || !phot_em_table.is_init()) + AbortWithMessage("Something went wrong with lookup table initialization"); + + auto of_dndt = ofstream{outfile_name + "_dndt"}; + of_dndt << std::setprecision(std::numeric_limits::digits10 + 1); + const auto coord_dndt = dndt_table.get_all_coordinates(); + for (const auto& cc : coord_dndt ) + of_dndt << cc << " " << dndt_table.interp(cc) << "\n"; + of_dndt.close(); + + auto of_phot_em = ofstream{outfile_name + "_phot_em"}; + of_phot_em << std::setprecision(std::numeric_limits::digits10 + 1); + + const auto wrapper = qs_photon_emission_table_wrapper{phot_em_table}; + wrapper.write_table_data(of_phot_em); + of_phot_em.close(); + + return; +} + +template +void +bw_pair_production_table_wrapper::write_table_data( + std::ofstream& of) const +{ + const auto how_many_x = this->m_table.get_how_many_x(); + const auto how_many_y = this->m_table.get_how_many_y(); + for (int i = 0; i < how_many_x; ++i){ + for (int j = 0; j < how_many_y; ++j){ + const auto xcoord = this->m_table.get_x_coord(i); + const auto ycoord = this->m_table.get_y_coord(j); + const auto val = this->m_table.get_val(i,j); + of << std::exp(xcoord) << " " << ycoord*std::exp(xcoord) + << " " << val << "\n"; + } + } +} + +template +void +qs_photon_emission_table_wrapper::write_table_data( + std::ofstream& of) const +{ + const auto how_many_x = this->m_table.get_how_many_x(); + const auto how_many_y = this->m_table.get_how_many_y(); + for (int i = 0; i < how_many_x; ++i){ + for (int j = 0; j < how_many_y; ++j){ + const auto xcoord = this->m_table.get_x_coord(i); + const auto ycoord = this->m_table.get_y_coord(j); + const auto val = this->m_table.get_val(i,j); + of << std::exp(xcoord) << " " << std::exp(ycoord)*std::exp(xcoord) + << " " << std::exp(val) << "\n"; + } + } +} diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index a189a8649d7..6e0e2ca5d1d 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -461,6 +461,7 @@ function(warpx_print_summary) message(" OPENPMD: ${WarpX_OPENPMD}") message(" QED: ${WarpX_QED}") message(" QED table generation: ${WarpX_QED_TABLE_GEN}") + message(" QED tools: ${WarpX_QED_TOOLS}") message(" SENSEI: ${WarpX_SENSEI}") message("") endfunction() diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake index a1e93a9e33c..9086421356f 100644 --- a/cmake/dependencies/PICSAR.cmake +++ b/cmake/dependencies/PICSAR.cmake @@ -14,18 +14,27 @@ function(find_picsar) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) # Enable or disable QED lookup tables generation - - # If table generation is enabled, enable or disable - # openMP support depending on WarpX_COMPUTE - if(WarpX_QED_TABLE_GEN) + if(WarpX_QED_TABLE_GEN OR WarpX_QED_TOOLS) set(PXRMP_QED_TABLEGEN ON CACHE INTERNAL "") - if(WarpX_COMPUTE STREQUAL OMP) - set(PXRMP_QED_OMP ON CACHE INTERNAL "") + else() + set(PXRMP_QED_TABLEGEN OFF CACHE INTERNAL "") + endif() + + # Enable or disable OpenMP for lookup tables generation + if(PXRMP_QED_TABLEGEN) + if(WarpX_QED_TABLES_GEN_OMP STREQUAL AUTO) + find_package(OpenMP REQUIRED CXX) + if(OpenMP_CXX_FOUND) + set(PXRMP_QED_OMP ON CACHE INTERNAL "") + else() + set(PXRMP_QED_OMP OFF CACHE INTERNAL "") + endif() + elseif(WarpX_QED_TABLES_GEN_OMP STREQUAL ON) + set(PXRMP_QED_OMP ON CACHE INTERNAL "") else() - set(PXRMP_QED_OMP OFF CACHE INTERNAL "") + set(PXRMP_QED_OMP OFF CACHE INTERNAL "") endif() else() - set(PXRMP_QED_TABLEGEN OFF CACHE INTERNAL "") set(PXRMP_QED_OMP OFF CACHE INTERNAL "") endif() From dde6f7b75b0edf3eed8c5a0f6508a8aefa66c776 Mon Sep 17 00:00:00 2001 From: Ryan Sandberg Date: Wed, 20 Mar 2024 14:18:25 -0700 Subject: [PATCH 019/121] fix variable add in picmi lab particle diagnostics (#4793) --- Python/pywarpx/picmi.py | 82 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index dfebd13eeb3..7b1daab412a 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2688,47 +2688,47 @@ def diagnostic_initialize_inputs(self): # --- Use a set to ensure that fields don't get repeated. variables = set() - - for dataname in self.data_list: - if dataname == 'position': - if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z - variables.add('x') - if pywarpx.geometry.dims == '3': - variables.add('y') - variables.add('z') - if pywarpx.geometry.dims == 'RZ': - variables.add('theta') - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: - if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): - raise RuntimeError( - f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" - f"chosen by dim={pywarpx.geometry.dims} in pywarpx." - ) - elif pywarpx.geometry.dims != '3' and dataname == 'y': - raise RuntimeError( - f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" - f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." - ) - elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': - raise RuntimeError( - f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." - f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." - ) - else: - variables.add(dataname) + if self.data_list is not None: + for dataname in self.data_list: + if dataname == 'position': + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname == 'fields': + variables.add('Ex') + variables.add('Ey') + variables.add('Ez') + variables.add('Bx') + variables.add('By') + variables.add('Bz') + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. From 1bdd4509c0c59aa72c80598fd42db020b39a31ea Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 21 Mar 2024 01:52:20 +0100 Subject: [PATCH 020/121] update Adastra doc (#4786) --- Tools/machines/adastra-cines/submit.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tools/machines/adastra-cines/submit.sh b/Tools/machines/adastra-cines/submit.sh index 15a2b292b58..e8bcb19975e 100644 --- a/Tools/machines/adastra-cines/submit.sh +++ b/Tools/machines/adastra-cines/submit.sh @@ -31,6 +31,14 @@ export MPICH_GPU_SUPPORT_ENABLED=1 # or, less invasive: export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor +# note +# On machines with similar architectures (Frontier, OLCF) these settings +# seem to prevent the following issue: +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors +export MPICH_SMP_SINGLE_COPY_MODE=NONE +export FI_CXI_RX_MATCH_MODE=software + # note # this environment setting is needed to avoid that rocFFT writes a cache in # the home directory, which does not scale. From 2b50441499e4397bf67426620c3d8cd56fcac30c Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 20 Mar 2024 20:04:17 -0700 Subject: [PATCH 021/121] Add function to get total weight of all particles in WarpXParticleContainer (#4795) --- .../ReducedDiags/ParticleNumber.cpp | 18 ++---------------- Source/Particles/WarpXParticleContainer.H | 1 + Source/Particles/WarpXParticleContainer.cpp | 16 ++++++++++------ .../Particles/WarpXParticleContainer.cpp | 4 ++++ 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp index 8c0377a3a82..7fe3f75d4ce 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp @@ -112,27 +112,13 @@ void ParticleNumber::ComputeDiags (int step) for (int i_s = 0; i_s < nSpecies; ++i_s) { // get WarpXParticleContainer class object - const auto & myspc = mypc.GetParticleContainer(i_s); + auto & myspc = mypc.GetParticleContainer(i_s); // Save total number of macroparticles for this species m_data[idx_first_species_macroparticles + i_s] = myspc.TotalNumberOfParticles(); - using PType = typename WarpXParticleContainer::SuperParticleType; - - // Reduction to compute sum of weights for this species - auto Wtot = ReduceSum( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> amrex::Real - { - return p.rdata(PIdx::w); - }); - - // MPI reduction - amrex::ParallelDescriptor::ReduceRealSum - (Wtot, amrex::ParallelDescriptor::IOProcessorNumber()); - // Save sum of particles weight for this species - m_data[idx_first_species_sum_weight + i_s] = Wtot; - + m_data[idx_first_species_sum_weight + i_s] = myspc.sumParticleWeight(false); // Increase total number of macroparticles and total weight (all species) m_data[idx_total_macroparticles] += m_data[idx_first_species_macroparticles + i_s]; diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7d2d5619da9..c100cc0bff4 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -280,6 +280,7 @@ public: /// This is needed when solving Poisson's equation with periodic boundary conditions. /// amrex::ParticleReal sumParticleCharge(bool local = false); + amrex::ParticleReal sumParticleWeight(bool local = false); std::array meanParticleVelocity(bool local = false); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 912dd3a5a40..46101c29e0b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1306,9 +1306,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) return rho; } -amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { +amrex::ParticleReal WarpXParticleContainer::sumParticleWeight(bool local) { - amrex::ParticleReal total_charge = 0.0; + amrex::ParticleReal total_weight = 0.0; ReduceOps reduce_op; ReduceData reduce_data(reduce_op); @@ -1328,11 +1328,15 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { } } - total_charge = get<0>(reduce_data.value()); + total_weight = get<0>(reduce_data.value()); + + if (!local) { ParallelDescriptor::ReduceRealSum(total_weight); } + return total_weight; +} + +amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { - if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } - total_charge *= this->charge; - return total_charge; + return this->sumParticleWeight(local) * this->charge; } std::array WarpXParticleContainer::meanParticleVelocity(bool local) { diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 07793a373f3..3feae5d8c90 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -109,6 +109,10 @@ void init_WarpXParticleContainer (py::module& m) &WarpXParticleContainer::TotalNumberOfParticles, py::arg("valid_particles_only"), py::arg("local") ) + .def("sum_particle_weight", + &WarpXParticleContainer::sumParticleWeight, + py::arg("local") + ) .def("sum_particle_charge", &WarpXParticleContainer::sumParticleCharge, py::arg("local") From 32772aad57900823af552cb4fa8a7fc24af5fe07 Mon Sep 17 00:00:00 2001 From: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Date: Wed, 20 Mar 2024 21:59:56 -0700 Subject: [PATCH 022/121] Electron current diagnostic for Hybrid solver (#4336) * JeFunctor to output electron current by subtracting ampere's current from total current in hybrid solver * add support for electron current diagnostic if hybrid solver is being used * add documentation for deposit_current parameter * add solver parameter for FieldDiagnostic passed in from Simulation object to help add in electron current diagnostics * Fix compilation errors * fix indentation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix include statement * always define a Je_fields_list in picmi rather than adding extra solver-dependent logic * fix typo * Unify signature in H/cpp * Add to Build Scripts * define m_deposit_current in header file * fix InterpolateMFForDiag function call with bool flag not to sum up modes as Hybrid-PIC only works for m=0 in RZ right now * calculate electron current for every dimension, bug fix * fix namespace issue, single dimension multifab (direction passed in when creating the Functor) * use member function to access private member var * semicolon * change m_dir to be an int from const int in declaration * fix attempted indexing on FAB * fix Je indexing in LinComb * added external current to electron current diag, code cleanup * output JeFunctor as a field_functor initialization loop * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix RZ openPMD functor checks for Je * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix quote issue * rename je to j_e to work with string parsing in converting from WarpX to OpenPMD * fix clang-tidy issue * remove comment * Reworked electron current to be a displacement current diagnostic, only implemented for HybridPICSolver right now. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update test to output new displacement current field * clang tidy fixes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * renamed picmi field quantity to J_displacement, comment clean-up, add in convert2RZmodes functionality. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update docs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix bug where old naming convention was left behind, plus a missing comma * Apply suggestions from code review Reduce code duplication Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove leftover comment block * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestions from code review fix skeleton code and comment * temporarily initialize curlB as nullptr in non-hybrid solver case * fix Jext staggering for subtraction * prepare for checksum * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update var name for checksum * fix OMP bug where tiling was attempted over uninitialized data * commenting out external current functionality to debug failing CI test * update checksum values * corrected staggering IntVects passed into interp function --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl Co-authored-by: Avigdor Veksler Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../theory/kinetic_fluid_hybrid_model.rst | 4 + .../Tests/ohm_solver_EM_modes/PICMI_inputs.py | 2 +- Python/pywarpx/picmi.py | 7 ++ .../Python_ohms_law_solver_EM_modes_1d.json | 6 +- .../ComputeDiagFunctors/CMakeLists.txt | 1 + .../ComputeDiagFunctors/JFunctor.H | 2 + .../ComputeDiagFunctors/JdispFunctor.H | 49 ++++++++ .../ComputeDiagFunctors/JdispFunctor.cpp | 106 ++++++++++++++++++ .../ComputeDiagFunctors/Make.package | 1 + Source/Diagnostics/FullDiagnostics.cpp | 34 +++++- .../HybridPICModel/HybridPICModel.H | 6 + Source/WarpX.H | 1 + 12 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H create mode 100644 Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 3fb8320c531..c517bd3b2f3 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -93,6 +93,10 @@ Implementation details Various verification tests of the hybrid model implementation can be found in the :ref:`examples section `. +.. note:: + WarpX's displacement current diagnostic is equivalent to electron current in + the kinetic-fluid hybrid model. + The kinetic-fluid hybrid extension mostly uses the same routines as the standard electromagnetic PIC algorithm with the only exception that the E-field is calculated from the above equation rather than it being updated from the full Maxwell-Ampere equation. The diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 5101e218129..6711331d538 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -268,7 +268,7 @@ def setup_run(self): name='field_diag', grid=self.grid, period=self.total_steps, - data_list=['B', 'E'], + data_list=['B', 'E', 'J_displacement'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_EM_modes_1d_plt', # warpx_format = 'openpmd', diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 7b1daab412a..3f7b139d4f2 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2247,11 +2247,13 @@ def diagnostic_initialize_inputs(self): E_fields_list = ['Er', 'Et', 'Ez'] B_fields_list = ['Br', 'Bt', 'Bz'] J_fields_list = ['Jr', 'Jt', 'Jz'] + J_displacement_fields_list = ['Jr_displacement', 'Jt_displacement', 'Jz_displacement'] A_fields_list = ['Ar', 'At', 'Az'] else: E_fields_list = ['Ex', 'Ey', 'Ez'] B_fields_list = ['Bx', 'By', 'Bz'] J_fields_list = ['Jx', 'Jy', 'Jz'] + J_displacement_fields_list = ['Jx_displacement', 'Jy_displacement', 'Jz_displacement'] A_fields_list = ['Ax', 'Ay', 'Az'] if self.data_list is not None: for dataname in self.data_list: @@ -2264,6 +2266,9 @@ def diagnostic_initialize_inputs(self): elif dataname == 'J': for field_name in J_fields_list: fields_to_plot.add(field_name.lower()) + elif dataname == 'J_displacement': + for field_name in J_displacement_fields_list: + fields_to_plot.add(field_name.lower()) elif dataname == 'A': for field_name in A_fields_list: fields_to_plot.add(field_name) @@ -2277,6 +2282,8 @@ def diagnostic_initialize_inputs(self): fields_to_plot.add(dataname) elif dataname in J_fields_list: fields_to_plot.add(dataname.lower()) + elif dataname in J_displacement_fields_list: + fields_to_plot.add(dataname.lower()) elif dataname.startswith('rho_'): # Adds rho_species diagnostic fields_to_plot.add(dataname) diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json index f5606cfee2f..5e42d7ec14b 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json @@ -5,7 +5,10 @@ "Bz": 256.0, "Ex": 1954.3267519602405, "Ey": 2363.4281756166347, - "Ez": 4873.508158589938 + "Ez": 4873.508158589938, + "jx_displacement": 914396602.5344071, + "jy_displacement": 762623912.4748704, + "jz_displacement": 2938492698.375407 }, "ions": { "particle_momentum_x": 1.6151135948675135e-19, @@ -15,3 +18,4 @@ "particle_weight": 4.220251350277737e+21 } } + diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt index 2ef6af16bfe..5e0eeaab73a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -6,6 +6,7 @@ foreach(D IN LISTS WarpX_DIMS) DivBFunctor.cpp DivEFunctor.cpp JFunctor.cpp + JdispFunctor.cpp RhoFunctor.cpp PartPerCellFunctor.cpp PartPerGridFunctor.cpp diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index 3080b500712..d9f9a1e82e0 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -19,6 +19,8 @@ public: to the output diagnostic MultiFab, mf_dst. * \param[in] convertRZmodes2cartesian (in cylindrical) whether to * sum all modes in mf_src before cell-centering into dst multifab. + * \param[in] deposit_current whether to deposit current. Used to output current in cases + where no field solve is used. * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ JFunctor (int dir, int lev, diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H new file mode 100644 index 00000000000..aa364f6da3b --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H @@ -0,0 +1,49 @@ +#ifndef WARPX_JDISPFUNCTOR_H_ +#define WARPX_JDISPFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +#include + +/** + * \brief Functor to cell-center MF for displacement current density and store + * result in mf_out. + */ +class +JdispFunctor final: public ComputeDiagFunctor +{ +public: + /** + * + * \param[in] dir direction of vector field to operate on + * \param[in] lev level of multifab. Used for averaging in rz. + * \param[in] crse_ratio for interpolating field values from the simulation Multifab, src_mf, + * to the output diagnostic MultiFab, mf_dst. + * \param[in] convertRZmodes2cartesian (in cylindrical) whether to + * sum all modes in mf_src before cell-centering into dst multifab. + * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. + */ + JdispFunctor (int dir, int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1); + /** \brief Cell-center m_mf_src and write the result in mf_dst. + * + * In cylindrical geometry, by default this functor average all components + * of a MultiFab and writes into one single component. + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered + * data is stored + */ + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; +private: + /** direction of the electron current density to save */ + int m_dir; + /** level on which mf_src is defined */ + int m_lev; + /** (for cylindrical) whether to average all modes into 1 comp */ + bool m_convertRZmodes2cartesian; +}; + +#endif // WARPX_JDISPFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp new file mode 100644 index 00000000000..a8051b33747 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -0,0 +1,106 @@ +/* This file is part of Warpx. + * + * Authors: Avigdor Veksler + * License: BSD-3-Clause-LBNL +*/ +#include "JdispFunctor.H" + +#include "WarpX.H" +#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" +#include "Particles/MultiParticleContainer.H" + +#include +#include +#include + +using namespace amrex; + +JdispFunctor::JdispFunctor (int dir, int lev, + amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) + : ComputeDiagFunctor(ncomp, crse_ratio), m_dir(dir), m_lev(lev), + m_convertRZmodes2cartesian(convertRZmodes2cartesian) +{ } + +void +JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const +{ + auto& warpx = WarpX::GetInstance(); + auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); + + /** pointer to source1 (total simulation current J) multifab */ + amrex::MultiFab* m_mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* m_mf_curlB; + [[maybe_unused]] amrex::MultiFab* m_mf_j_external; + if (hybrid_pic_model) { + /** pointer to source2 (current calculated from Ampere's Law, Jamp) multifab */ + m_mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); + /** pointer to source3 (external currents, Jext) multifab */ + m_mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + } else { + // To finish this implementation, we need to implement a method to + // calculate (∇ x B). + m_mf_curlB = nullptr; // Remove when curlB implemented + WARPX_ABORT_WITH_MESSAGE( + "Displacement current diagnostic is only implemented for the HybridPICModel."); + + // Skeleton for future implementation for solvers other than HybridPIC. + // Get curlB multifab + + // Divide curlB multifab by mu0 to get units of current + // m_mf_curlB->mult(1.0/PhysConsts::mu0) + } + // A Jdisp multifab is generated to hold displacement current. + amrex::MultiFab Jdisp( m_mf_j->boxArray(), m_mf_j->DistributionMap(), 1, m_mf_j->nGrowVect() ); + Jdisp.setVal(0); + + // J_displacement = curl x B / mu0 - J + amrex::MultiFab::LinComb( + Jdisp, 1, *m_mf_curlB, 0, + -1, *m_mf_j, 0, 0, 1, Jdisp.nGrowVect() + ); + + // Subtract the interpolated j_external value from j_displacement. + if (hybrid_pic_model) { + // Index type required for interpolating Jext from their respective + // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement + // locations. The staggering of J_displacement is the same as the + // staggering for J, so we use J_stag as the interpolation map. + // For interp to work below, the indices of the undefined dimensions + // must match. We set them as (1,1,1). + amrex::GpuArray Jext_IndexType = {1, 1, 1}; + amrex::GpuArray J_IndexType = {1, 1, 1}; + amrex::IntVect Jext_stag = m_mf_j_external->ixType().toIntVect(); + amrex::IntVect J_stag = m_mf_j->ixType().toIntVect(); + + // Index types for the dimensions simulated are overwritten. + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + Jext_IndexType[idim] = Jext_stag[idim]; + J_IndexType[idim] = J_stag[idim]; + } + + // Parameters for `interp` that maps from Jext to J. + // The "coarsening is just 1 i.e. no coarsening" + amrex::GpuArray const& coarsen = {1, 1, 1}; + + // Loop through the grids, and over the tiles within each grid to + // subtract the interpolated Jext from J_displacement. +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(Jdisp, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + + Array4 const& Jdisp_arr = Jdisp.array(mfi); + Array4 const& Jext = m_mf_j_external->const_array(mfi); + + // Loop over cells and update the Jdisp MultiFab + amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Interpolate Jext to the staggering of J + auto const jext_interp = ablastr::coarsen::sample::Interp(Jext, Jext_IndexType, J_IndexType, coarsen, i, j, k, 0); + Jdisp_arr(i, j, k, 0) -= jext_interp; + }); + } + } + + InterpolateMFForDiag(mf_dst, Jdisp, dcomp, warpx.DistributionMap(m_lev), + m_convertRZmodes2cartesian); +} diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index 9c329263b0f..fd1624b8708 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -4,6 +4,7 @@ CEXE_sources += PartPerGridFunctor.cpp CEXE_sources += DivBFunctor.cpp CEXE_sources += DivEFunctor.cpp CEXE_sources += JFunctor.cpp +CEXE_sources += JdispFunctor.cpp CEXE_sources += RhoFunctor.cpp CEXE_sources += BackTransformFunctor.cpp CEXE_sources += BackTransformParticleFunctor.cpp diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index d93c67fe893..27fe7e48c66 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -3,6 +3,7 @@ #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" #include "ComputeDiagFunctors/JFunctor.H" +#include "ComputeDiagFunctors/JdispFunctor.H" #include "ComputeDiagFunctors/PartPerCellFunctor.H" #include "ComputeDiagFunctors/PartPerGridFunctor.H" #include "ComputeDiagFunctors/ParticleReductionFunctor.H" @@ -266,6 +267,24 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) if (update_varnames) { AddRZModesToOutputNames(std::string("jz"), ncomp); } + } else if ( m_varnames_fields[comp] == "jr_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jr_displacement"), ncomp); + } + } else if ( m_varnames_fields[comp] == "jt_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jt_displacement"), ncomp); + } + } else if ( m_varnames_fields[comp] == "jz_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jz_displacement"), ncomp); + } } else if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, @@ -636,6 +655,8 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if ( m_varnames[comp] == "jz_displacement" ) { + m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ @@ -677,11 +698,17 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jt" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if (m_varnames[comp] == "jr_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true); + } else if (m_varnames[comp] == "jt_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); - } else { + } + else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } #else @@ -700,11 +727,16 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if ( m_varnames[comp] == "jx_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jy_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); } else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); } #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 5e4c6382bfc..7d477b4e5fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -200,6 +200,12 @@ public: return current_fp_ampere[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab* + get_pointer_current_fp_external (int lev, int direction) const + { + return current_fp_external[lev][direction].get(); + } + [[nodiscard]] amrex::MultiFab* get_pointer_electron_pressure_fp (int lev) const { diff --git a/Source/WarpX.H b/Source/WarpX.H index 6526171ff67..d7b7266d7b0 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -131,6 +131,7 @@ public: MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } + [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} #ifdef AMREX_USE_EB amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} From daa8b3b238ff436331d6587b85b27337c15ce46d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 21 Mar 2024 07:48:40 -0700 Subject: [PATCH 023/121] Update spacecraft charging test (#4792) * Update spacecraft charging test * Apply suggestions from code review * Apply suggestions from code review * Make calculation of `rho_integral` faster * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py * Update checksum --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../spacecraft_charging/PICMI_inputs_rz.py | 20 ++++----- .../spacecraft_charging/analysis.py | 4 +- .../benchmarks_json/spacecraft_charging.json | 42 +++++++++---------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 3de56a93710..6eefdca942d 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -25,7 +25,8 @@ class SpaceChargeFieldCorrector(object): """ Class used by the callback functions to calculate the - correct charge on the spacecraft at each initialisation. + correct field around the spacecraft, at each timestep + (taking into account the charge that has been collected on the spacecraft) """ def __init__(self): self.saved_first_iteration_fields = False @@ -46,12 +47,12 @@ def correct_space_charge_fields(self, q=None): q = compute_actual_charge_on_spacecraft() # Correct fields so as to recover the actual charge - Er = ExWrapper(include_ghosts=True)[:,:] - Er[...] = Er[...]+(q - q_v)*self.normalized_Er[...] - Ez = EzWrapper(include_ghosts=True)[:,:] - Ez[...] += (q - q_v)*self.normalized_Ez[...] - phi = PhiFPWrapper(include_ghosts=True)[:,:] - phi[...] += (q - q_v)*self.normalized_phi[...] + Er = ExWrapper(include_ghosts=True) + Er[...] += (q - q_v)*self.normalized_Er + Ez = EzWrapper(include_ghosts=True) + Ez[...] += (q - q_v)*self.normalized_Ez + phi = PhiFPWrapper(include_ghosts=True) + phi[...] += (q - q_v)*self.normalized_phi self.spacecraft_potential += (q - q_v)*self.spacecraft_capacitance sim.extension.warpx.set_potential_on_eb( "%f" %self.spacecraft_potential ) print('Setting potential to %f' %self.spacecraft_potential) @@ -111,10 +112,7 @@ def compute_virtual_charge_on_spacecraft(): # Compute integral of rho over volume of the domain # (i.e. total charge of the plasma particles) - rho_integral = 0.0 - for k in range(1, nz-1): - for i in range(1, nr-1): - rho_integral += rho[i,k] * r[i] * dr * dz + rho_integral = (rho[1:nr-1,1:nz-1] * r[1:nr-1,np.newaxis]).sum()*dr*dz # Due to an oddity in WarpX (which will probably be solved later) # we need to multiply `rho` by `-epsilon_0` to get the correct charge diff --git a/Examples/Physics_applications/spacecraft_charging/analysis.py b/Examples/Physics_applications/spacecraft_charging/analysis.py index 10786baab61..ef75fd1a10a 100755 --- a/Examples/Physics_applications/spacecraft_charging/analysis.py +++ b/Examples/Physics_applications/spacecraft_charging/analysis.py @@ -61,8 +61,8 @@ def func(x, v0, tau): print('v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) -tolerance_v0=0.01 -tolerance_tau=0.01 +tolerance_v0=0.04 +tolerance_tau=0.04 print("tolerance for v0 = "+ str(tolerance_v0 *100) + '%') print("tolerance for tau = "+ str(tolerance_tau*100) + '%') diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/spacecraft_charging.json index d9f753e1df2..2994d4feeae 100644 --- a/Regression/Checksum/benchmarks_json/spacecraft_charging.json +++ b/Regression/Checksum/benchmarks_json/spacecraft_charging.json @@ -1,30 +1,28 @@ { "lev=0": { - "Er": 75713.05000099652, - "Ez": 75260.78239853957, - "phi": 55650.30604185804, - "rho": 1.4793075271598396e-06, - "rho_electrons": 6.506538129003745e-06, - "rho_protons": 6.98347902659172e-06 + "Er": 72926.92774219153, + "Ez": 71877.43945667242, + "phi": 58270.32861572235, + "rho": 1.460951374255109e-06, + "rho_electrons": 6.501141119196457e-06, + "rho_protons": 6.992817056027598e-06 }, "electrons": { - "particle_position_x": 38158.7364935527, - "particle_position_y": 37779.25499255196, - "particle_position_z": 45010.371467374425, - "particle_momentum_x": 8.27307207197173e-20, - "particle_momentum_y": 8.264475255806164e-20, - "particle_momentum_z": 8.271327169054914e-20, - "particle_weight": 1140673608016.2212 + "particle_position_x": 38094.07563382932, + "particle_position_y": 37781.48683071032, + "particle_position_z": 44952.3745721341, + "particle_momentum_x": 8.278426203619687e-20, + "particle_momentum_y": 8.272126720129096e-20, + "particle_momentum_z": 8.268559239607837e-20, + "particle_weight": 1140843527312.664 }, "protons": { - "particle_position_x": 751407.372588289, - "particle_position_y": 751687.788498272, - "particle_position_z": 644420.0485148785, - "particle_momentum_x": 1.468116154656724e-17, - "particle_momentum_y": 1.4650318746367807e-17, - "particle_momentum_z": 1.1638654342620123e-17, - "particle_weight": 1175692137613.312 + "particle_position_x": 751412.1033107714, + "particle_position_y": 751687.9554423956, + "particle_position_z": 644516.6066265699, + "particle_momentum_x": 1.4698270572291615e-17, + "particle_momentum_y": 1.466699162579312e-17, + "particle_momentum_z": 1.162881798230096e-17, + "particle_weight": 1176011216106.0706 } } - - From 1381da3682cba2bc01a3499a0bdfdfd1138dad1a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 22 Mar 2024 09:26:18 -0700 Subject: [PATCH 024/121] Document the function `add_applied_field` in PICMI (#4794) --- Docs/source/usage/python.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 639b57d219e..3aba5f165de 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -35,7 +35,7 @@ Simulation and Grid Setup ------------------------- .. autoclass:: pywarpx.picmi.Simulation - :members: step, add_species, add_laser, write_input_file + :members: step, add_species, add_laser, add_applied_field, write_input_file .. autoclass:: pywarpx.picmi.Cartesian3DGrid @@ -69,6 +69,8 @@ which can be used directly inside any PICMI script. The values are in SI units. Applied fields -------------- +Instances of the classes below need to be passed to the method `add_applied_field` of the `Simulation` class. + .. autoclass:: pywarpx.picmi.AnalyticInitialField .. autoclass:: pywarpx.picmi.ConstantAppliedField From e39ead78d0416268fcba3365ad9657cf37192ce1 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:12:11 -0700 Subject: [PATCH 025/121] Docs: update hybrid-pic description and parameters (#4796) * Docs: update hybrid-pic description and parameters * also update docstring in picmi --- .../theory/kinetic_fluid_hybrid_model.rst | 18 +++++++++++------- Docs/source/usage/parameters.rst | 3 +++ Python/pywarpx/picmi.py | 3 +++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index c517bd3b2f3..b4f494d8382 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -73,13 +73,13 @@ If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above en_e\vec{E} = -\vec{J}_e\times\vec{B}-\nabla\cdot{\overleftrightarrow P}_e+\vec{R}_e. Making the further simplifying assumptions that the electron pressure is isotropic and that -the electron drag term can be written as a simple resistance -i.e. :math:`\vec{R}_e = en_e\vec{\eta}\cdot\vec{J}`, brings us to the implemented form of +the electron drag term can be written using a simple resistivity (:math:`\eta`) and hyper-resistivity (:math:`\eta_h`) +i.e. :math:`\vec{R}_e = en_e(\eta-\eta_h \nabla^2)\vec{J}`, brings us to the implemented form of Ohm's law: .. math:: - \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\vec{\eta}\cdot\vec{J}. + \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\eta\vec{J}-\eta_h \nabla^2\vec{J}. Lastly, if an electron temperature is given from which the electron pressure can be calculated, the model is fully constrained and can be evolved given initial @@ -93,10 +93,6 @@ Implementation details Various verification tests of the hybrid model implementation can be found in the :ref:`examples section `. -.. note:: - WarpX's displacement current diagnostic is equivalent to electron current in - the kinetic-fluid hybrid model. - The kinetic-fluid hybrid extension mostly uses the same routines as the standard electromagnetic PIC algorithm with the only exception that the E-field is calculated from the above equation rather than it being updated from the full Maxwell-Ampere equation. The @@ -171,5 +167,13 @@ input parameters, :math:`T_{e0}`, :math:`n_0` and :math:`\gamma` using The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. +Electron current +^^^^^^^^^^^^^^^^ + +WarpX's displacement current diagnostic can be used to output the electron current in +the kinetic-fluid hybrid model since in the absence of kinetic electrons, and under +the assumption of zero displacement current, that diagnostic simply calculates the +hybrid model's electron current. + .. bibliography:: :keyprefix: kfhm- diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index bf66c24ccb9..1a3e9e63064 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2263,6 +2263,9 @@ Maxwell solver: kinetic-fluid hybrid * ``hybrid_pic_model.plasma_resistivity(rho,J)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. +* ``hybrid_pic_model.plasma_hyper_resistivity`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma hyper-resistivity in :math:`\Omega m^3`. + * ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the external current (on the grid) in :math:`A/m^2`. diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 3f7b139d4f2..9b64c6e8ecf 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1134,6 +1134,9 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): plasma_resistivity: float or str Value or expression to use for the plasma resistivity. + plasma_hyper_resistivity: float or str + Value or expression to use for the plasma hyper-resistivity. + substeps: int, default=100 Number of substeps to take when updating the B-field. From 964c77e193bfc18a9d9f9ffd4be5e2434723a4ec Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 23 Mar 2024 00:41:53 -0700 Subject: [PATCH 026/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4800) * AMReX: Weekly Update * pyAMReX: Weekly Update * AMReX: Weekly Update --------- Co-authored-by: roelof-groenewald --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 1c384183fbd..feb779d6ad4 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 155f1f4c3f4fa03528d38d2769beb4b35b38042c && cd - + cd ../amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 138cf90534f..d5ddbe2614d 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 155f1f4c3f4fa03528d38d2769beb4b35b38042c +branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 56a7fbc8f10..69b9f0d4b4d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 155f1f4c3f4fa03528d38d2769beb4b35b38042c +branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 54b1cc45322..4c9a7bcb906 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "155f1f4c3f4fa03528d38d2769beb4b35b38042c" +set(WarpX_amrex_branch "a8633589c1fae5b3fda6deca86fba79878a8cb54" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index eaa0fc016cd..3242747b6f5 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "7cd4a4220bc3fc523cd0d78f7320909aa2feb47a" +set(WarpX_pyamrex_branch "e028359d5679e0775b4a8a36332c5d1d37fc7506" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 1bf6e96d236..081efe21626 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 155f1f4c3f4fa03528d38d2769beb4b35b38042c && cd - +cd amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From aa8df1cf1aa576d4e10e68cdedd3e66e132cc1be Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Sat, 23 Mar 2024 03:44:24 -0700 Subject: [PATCH 027/121] Fix a number of compilation warnings (#4801) * Fix a number of compilation warnings * remove `m_` from non-member variables --- .../ComputeDiagFunctors/JdispFunctor.cpp | 34 +++++++++---------- Source/EmbeddedBoundary/ParticleScraper.H | 2 +- .../Particles/PhysicalParticleContainer.cpp | 4 ++- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index a8051b33747..373004d4633 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -27,19 +27,16 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto& warpx = WarpX::GetInstance(); auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); - /** pointer to source1 (total simulation current J) multifab */ - amrex::MultiFab* m_mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); - amrex::MultiFab* m_mf_curlB; - [[maybe_unused]] amrex::MultiFab* m_mf_j_external; + /** pointer to total simulation current (J) multifab */ + amrex::MultiFab* mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* mf_curlB; if (hybrid_pic_model) { - /** pointer to source2 (current calculated from Ampere's Law, Jamp) multifab */ - m_mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); - /** pointer to source3 (external currents, Jext) multifab */ - m_mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + /** pointer to current calculated from Ampere's Law (Jamp) multifab */ + mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); } else { // To finish this implementation, we need to implement a method to // calculate (∇ x B). - m_mf_curlB = nullptr; // Remove when curlB implemented + mf_curlB = nullptr; // Remove when curlB implemented WARPX_ABORT_WITH_MESSAGE( "Displacement current diagnostic is only implemented for the HybridPICModel."); @@ -47,20 +44,23 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff // Get curlB multifab // Divide curlB multifab by mu0 to get units of current - // m_mf_curlB->mult(1.0/PhysConsts::mu0) + // mf_curlB->mult(1.0/PhysConsts::mu0) } // A Jdisp multifab is generated to hold displacement current. - amrex::MultiFab Jdisp( m_mf_j->boxArray(), m_mf_j->DistributionMap(), 1, m_mf_j->nGrowVect() ); + amrex::MultiFab Jdisp( mf_j->boxArray(), mf_j->DistributionMap(), 1, mf_j->nGrowVect() ); Jdisp.setVal(0); // J_displacement = curl x B / mu0 - J amrex::MultiFab::LinComb( - Jdisp, 1, *m_mf_curlB, 0, - -1, *m_mf_j, 0, 0, 1, Jdisp.nGrowVect() + Jdisp, 1, *mf_curlB, 0, + -1, *mf_j, 0, 0, 1, Jdisp.nGrowVect() ); - // Subtract the interpolated j_external value from j_displacement. if (hybrid_pic_model) { + // Subtract the interpolated j_external value from j_displacement. + /** pointer to external currents (Jext) multifab */ + amrex::MultiFab* mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + // Index type required for interpolating Jext from their respective // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement // locations. The staggering of J_displacement is the same as the @@ -69,8 +69,8 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff // must match. We set them as (1,1,1). amrex::GpuArray Jext_IndexType = {1, 1, 1}; amrex::GpuArray J_IndexType = {1, 1, 1}; - amrex::IntVect Jext_stag = m_mf_j_external->ixType().toIntVect(); - amrex::IntVect J_stag = m_mf_j->ixType().toIntVect(); + amrex::IntVect Jext_stag = mf_j_external->ixType().toIntVect(); + amrex::IntVect J_stag = mf_j->ixType().toIntVect(); // Index types for the dimensions simulated are overwritten. for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -90,7 +90,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff for ( MFIter mfi(Jdisp, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { Array4 const& Jdisp_arr = Jdisp.array(mfi); - Array4 const& Jext = m_mf_j_external->const_array(mfi); + Array4 const& Jext = mf_j_external->const_array(mfi); // Loop over cells and update the Jdisp MultiFab amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 7d6934cbb9c..d058de7e48f 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -189,7 +189,7 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t if (phi_value < 0.0) { int ic, jc, kc; // Cell-centered indices - int nodal; + [[maybe_unused]] int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 160eac0d19c..06154958dde 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -148,7 +148,9 @@ namespace AMREX_GPU_HOST_DEVICE PDim3(const amrex::XDim3& a): - x{a.x}, y{a.y}, z{a.z} + x{static_cast(a.x)}, + y{static_cast(a.y)}, + z{static_cast(a.z)} {} AMREX_GPU_HOST_DEVICE From 91f3ff1be27f1761b069c78399e8cac3fe5316be Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Sun, 24 Mar 2024 18:16:54 -0700 Subject: [PATCH 028/121] Clean up BeamRelevant with TypeMultiplier (#4798) * Clean up BeamRelevant with TypeMultiplier The BeamRelevant reduced diagnostics had a lot of code where `amrex::ReduceOps` were constructed from lists of types. It is hard to see immediately how many types are in that list. The `TypeMultiplier` utility can reduce this to make it more legible and this was done here. * Change length variable to constexpr size_t The variables `num_red_ops_*` are used for the TypeMultiplier and also for the length of the output vectors for the reduced diagnostics. This commit changes their type from `const int` to `constexpr size_t` because this is more appropriate for a vector length, and with constexpr we indicate that the value is constant and can be evaluated at compile time already. --- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 78 +++++++++---------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index fbe67ba6fc5..b78a7713abc 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -198,14 +199,14 @@ void BeamRelevant::ComputeDiags (int step) using PType = typename WarpXParticleContainer::SuperParticleType; - amrex::ReduceOps reduce_ops; - auto r = amrex::ParticleReduce>( + // number of reduction operations in first concurrent batch + constexpr size_t num_red_ops_1 = 8; + TypeMultiplier reduce_ops_1; + using ReducedDataT1 = TypeMultiplier; + + auto r1 = amrex::ParticleReduce( myspc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple - + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> ReducedDataT1::Type { const ParticleReal p_ux = p.rdata(PIdx::ux); const ParticleReal p_uy = p.rdata(PIdx::uy); @@ -242,18 +243,17 @@ void BeamRelevant::ComputeDiags (int step) p_ux_mean, p_uy_mean, p_uz_mean, p_gm_mean}; }, - reduce_ops); - - std::vector values_per_rank_1st = { - amrex::get<0>(r), // w - amrex::get<1>(r), // x_mean - amrex::get<2>(r), // y_mean - amrex::get<3>(r), // z_mean - amrex::get<4>(r), // ux_mean - amrex::get<5>(r), // uy_mean - amrex::get<6>(r), // uz_mean - amrex::get<7>(r), // gm_mean - }; + reduce_ops_1); + + std::vector values_per_rank_1st(num_red_ops_1); + + /* contains in this order: + * w, x_mean, y_mean, z_mean + * ux_mean, uy_mean, uz_mean, gm_mean + */ + amrex::constexpr_for<0, num_red_ops_1> ([&](auto i) { + values_per_rank_1st[i] = amrex::get(r1); + }); // reduced sum over mpi ranks (allreduce) amrex::ParallelAllReduce::Sum @@ -275,16 +275,15 @@ void BeamRelevant::ComputeDiags (int step) return; } - amrex::ReduceOps reduce_ops2; + // number of reduction operations in second concurrent batch + constexpr size_t num_red_ops_2 = 11; + + TypeMultiplier reduce_ops2; + using ReducedDataT2 = TypeMultiplier; - auto r2 = amrex::ParticleReduce>( + auto r2 = amrex::ParticleReduce( myspc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple - + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> ReducedDataT2::Type { const ParticleReal p_ux = p.rdata(PIdx::ux); const ParticleReal p_uy = p.rdata(PIdx::uy); @@ -336,19 +335,18 @@ void BeamRelevant::ComputeDiags (int step) }, reduce_ops2); - std::vector values_per_rank_2nd = { - amrex::get<0>(r2), // x_ms - amrex::get<1>(r2), // y_ms - amrex::get<2>(r2), // z_ms - amrex::get<3>(r2), // ux_ms - amrex::get<4>(r2), // uy_ms - amrex::get<5>(r2), // uz_ms - amrex::get<6>(r2), // gm_ms - amrex::get<7>(r2), // xux - amrex::get<8>(r2), // yuy - amrex::get<9>(r2), // zuz - amrex::get<10>(r2) // charge - }; + std::vector values_per_rank_2nd(num_red_ops_2); + + /* contains in this order: + * x_ms, y_ms, z_ms + * ux_ms, uy_ms, uz_ms, + * gm_ms + * xux, yuy, zuz, + * charge + */ + amrex::constexpr_for<0, num_red_ops_2> ([&](auto i) { + values_per_rank_2nd[i] = amrex::get(r2); + }); // reduced sum over mpi ranks (reduce to IO rank) ParallelDescriptor::ReduceRealSum From d7b1abac8afa924efd012af6752fac22475752f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Ordyna?= Date: Mon, 25 Mar 2024 02:19:13 +0100 Subject: [PATCH 029/121] Fix ParticleHistogram2D (#4779) * fix unfreed host buffer and missing iteration and series close in ParticleHistogram2D * More efficient resize Co-authored-by: Weiqun Zhang --------- Co-authored-by: Axel Huebl Co-authored-by: Weiqun Zhang --- Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index bcc22eac598..0d696f8b7bc 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -138,7 +138,7 @@ void ParticleHistogram2D::ComputeDiags (int step) Array tlo{0,0}; // lower bounds Array thi{m_bin_num_abs-1, m_bin_num_ord-1}; // inclusive upper bounds amrex::TableData d_data_2D(tlo, thi); - m_h_data_2D = amrex::TableData (tlo, thi, The_Pinned_Arena()); + m_h_data_2D.resize(tlo, thi, The_Pinned_Arena()); auto const& h_table_data = m_h_data_2D.table(); // Initialize data on the host @@ -316,6 +316,8 @@ void ParticleHistogram2D::WriteToFile (int step) const {static_cast(m_bin_num_ord), static_cast(m_bin_num_abs)}); series.flush(); + i.close(); + series.close(); #else amrex::ignore_unused(step); WARPX_ABORT_WITH_MESSAGE("ParticleHistogram2D: Needs openPMD-api compiled into WarpX, but was not found!"); From f49a63f038cf081ca8ea93420c0ddffb3279b918 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 25 Mar 2024 16:03:05 -0700 Subject: [PATCH 030/121] Remove GPUClock Cost Function (#4802) Removing the GPUClock cost function due to the following reasons. Incomplete Implementation: The implementation is only added to selected kernels. The implementation is not generalized to work with varying occupancy of different kernels, even if it were used in all kernels. The implementation is verbose. Unused: Our host-side timer implementation was in the last years extended to synchronize kernels at minimal overhead cost. This and heuristic is actually used. Research scope shifted: In the last years, we realized that we do not need more precise scalar cost functions, but instead vector cost functions to build better load balance performance models from. Costly when used: The implementation uses an atomic add of each kernel, instead of, e.g., just using one per warp. This adds severe memory bandwidth strain. Costly, even if not used: The implementation adds about 4 registers unnecessary to all instrumented GPU kernels once compiled in (by default). --- CMakeLists.txt | 8 - Docs/source/developers/testing.rst | 2 +- Docs/source/install/cmake.rst | 1 - Docs/source/usage/parameters.rst | 6 +- GNUmakefile | 1 - Python/pywarpx/picmi.py | 2 +- Source/Make.WarpX | 5 - .../Particles/Deposition/ChargeDeposition.H | 59 +---- .../Particles/Deposition/CurrentDeposition.H | 210 +----------------- Source/Particles/WarpXParticleContainer.cpp | 114 ++++------ Source/Utils/WarpXAlgorithmSelection.H | 5 +- Source/Utils/WarpXAlgorithmSelection.cpp | 1 - Source/WarpX.H | 2 +- Source/WarpX.cpp | 4 - Source/ablastr/particles/DepositCharge.H | 17 +- Tools/PerformanceTests/summit.py | 2 +- WarpXConfig.cmake | 1 - cmake/WarpXFunctions.cmake | 1 - 18 files changed, 65 insertions(+), 376 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1d5dca6e3a..55fae3db78d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,10 +69,6 @@ include(CMakeDependentOption) option(WarpX_APP "Build the WarpX executable application" ON) option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) option(WarpX_EB "Embedded boundary support" OFF) -cmake_dependent_option(WarpX_GPUCLOCK - "Add GPU kernel timers (cost function)" ON - "WarpX_COMPUTE STREQUAL CUDA OR WarpX_COMPUTE STREQUAL HIP" - OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) @@ -510,10 +506,6 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_RZ WARPX_ZINDEX=1) endif() - if(WarpX_GPUCLOCK) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_GPUCLOCK) - endif() - if(WarpX_OPENPMD) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_OPENPMD) endif() diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 6dcc1886e12..c6a09d970df 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -90,7 +90,7 @@ The content of this directory will look like the following (possibly including b $ ls ./test_dir/rt-WarpX/WarpX-tests/2021-04-30/pml_x_yee/ analysis_pml_yee.py # Python analysis script inputs_2d # input file - main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.GPUCLOCK.ex # executable + main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.ex # executable pml_x_yee.analysis.out # Python analysis output pml_x_yee.err.out # error output pml_x_yee.make.out # build output diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 7ff8986cf36..2e66c95a4d6 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -89,7 +89,6 @@ CMake Option Default & Values Descr ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WarpX_DIMS`` **3**/2/1/RZ Simulation dimensionality. Use ``"1;2;RZ;3"`` for all. ``WarpX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) -``WarpX_GPUCLOCK`` **ON**/OFF Add GPU kernel timers (cost function, +4 registers/kernel) ``WarpX_IPO`` ON/**OFF** Compile WarpX with interprocedural optimization (aka LTO) ``WarpX_LIB`` ON/**OFF** Build WarpX as a library, e.g., for PICMI Python ``WarpX_MPI`` **ON**/OFF Multi-node support (message-passing) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 1a3e9e63064..fc6d338fc1c 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -557,7 +557,7 @@ Distribution across MPI ranks and parallelization For example, if there are 4 boxes per rank and `load_balance_knapsack_factor=2`, no more than 8 boxes can be assigned to any rank. -* ``algo.load_balance_costs_update`` (`heuristic` or `timers` or `gpuclock`) optional (default `timers`) +* ``algo.load_balance_costs_update`` (``heuristic`` or ``timers``) optional (default ``timers``) If this is `heuristic`: load balance costs are updated according to a measure of particles and cells assigned to each box of the domain. The cost :math:`c` is computed as @@ -574,10 +574,6 @@ Distribution across MPI ranks and parallelization If this is `timers`: costs are updated according to in-code timers. - If this is `gpuclock`: [**requires to compile with option** ``-DWarpX_GPUCLOCK=ON``] - costs are measured as (max-over-threads) time spent in current deposition - routine (only applies when running on GPUs). - * ``algo.costs_heuristic_particles_wt`` (`float`) optional Particle weight factor used in `Heuristic` strategy for costs update; if running on GPU, the particle weight is set to a value determined from single-GPU tests on Summit, diff --git a/GNUmakefile b/GNUmakefile index 7ddded50483..863d37dd56f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -29,7 +29,6 @@ USE_GPU = FALSE EBASE = main -USE_GPUCLOCK = TRUE USE_PYTHON_MAIN = FALSE USE_SENSEI_INSITU = FALSE diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 9b64c6e8ecf..5902b958bc9 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1825,7 +1825,7 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_load_balance_knapsack_factor: float, default=1.24 (See documentation) - warpx_load_balance_costs_update: {'heuristic' or 'timers' or 'gpuclock'}, optional + warpx_load_balance_costs_update: {'heuristic' or 'timers'}, optional (See documentation) warpx_costs_heuristic_particles_wt: float, optional diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 6c55372d199..4662dc18abd 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -214,11 +214,6 @@ ifeq ($(USE_HDF5),TRUE) DEFINES += -DWARPX_USE_HDF5 endif -ifeq ($(USE_GPUCLOCK),TRUE) - USERSuffix := $(USERSuffix).GPUCLOCK - DEFINES += -DWARPX_USE_GPUCLOCK -endif - # job_info support CEXE_sources += AMReX_buildInfo.cpp INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index d0822789015..8ff712eefa2 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -33,8 +33,6 @@ * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost: Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, @@ -42,20 +40,14 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const int* ion_lev, amrex::FArrayBox& rho_fab, long np_to_deposit, - const std::array& dx, + const std::array& dx, const std::array xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -87,21 +79,9 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into rho_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0.; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif // --- Get particle quantities amrex::Real wq = q*wp[ip]*invvol; if (do_ionization){ @@ -202,13 +182,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #ifndef WARPX_DIM_RZ amrex::ignore_unused(n_rz_azimuthal_modes); @@ -230,8 +203,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. * \param a_bins * \param box * \param geom @@ -245,13 +216,11 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio amrex::FArrayBox& rho_fab, const amrex::IntVect& ix_type, const long np_to_deposit, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, const int n_rz_azimuthal_modes, - amrex::Real* cost, - const long load_balance_costs_update_algo, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -264,7 +233,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) - amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); + amrex::ignore_unused(ix_type, a_bins, box, geom, a_tbox_max_size, bin_size); #endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer @@ -299,14 +268,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into rho_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0.; - } -#endif - #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); @@ -394,11 +355,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio { const unsigned int ip = permutation[ip_orig]; -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif // --- Get particle quantities amrex::Real wq = q*wp[ip]*invvol; if (do_ionization){ @@ -506,13 +462,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif // defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #ifndef WARPX_DIM_RZ amrex::ignore_unused(n_rz_azimuthal_modes); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 8d4c5b6a5f4..e722839c9c6 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -287,8 +287,6 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionShapeN (const GetParticlePosition& GetPosition, @@ -306,9 +304,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, const std::array& xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex::literals; @@ -316,10 +312,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -354,22 +346,9 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jz_type = jz_fab.box().type(); // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - amrex::ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); @@ -395,13 +374,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -424,8 +396,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, @@ -445,19 +415,13 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, const std::array& xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real* cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex::literals; #if !defined(WARPX_DIM_RZ) amrex::ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -490,22 +454,9 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, amrex::IntVect const jz_type = jz_fab.box().type(); // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - amrex::ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); @@ -539,13 +490,6 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -570,8 +514,6 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, @@ -590,8 +532,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::Dim3 lo, amrex::Real q, int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -616,13 +556,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); const auto domain = geom.Domain(); @@ -651,11 +584,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::launch( nblocks, threads_per_block, shared_mem_bytes, amrex::Gpu::gpuStream(), [=] AMREX_GPU_DEVICE () noexcept { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif const int bin_id = blockIdx.x; const unsigned int bin_start = offsets_ptr[bin_id]; const unsigned int bin_stop = offsets_ptr[bin_id+1]; @@ -743,18 +671,11 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, __syncthreads(); addLocalToGlobal(tbox_z, jz_arr, jz_buff); }); -#if defined(WARPX_USE_GPUCLOCK) - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #else // not using hip/cuda // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -782,8 +703,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -802,9 +721,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, std::array xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real * const cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex; using namespace amrex::literals; @@ -813,10 +730,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -857,22 +770,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - // --- Get particle quantities Real const gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq @@ -1127,13 +1027,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -1159,8 +1052,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, @@ -1180,23 +1071,17 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real * const cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex; #if !defined(WARPX_DIM_RZ) ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -1236,22 +1121,9 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - #if !defined(WARPX_DIM_3D) constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); @@ -1526,13 +1398,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -1560,8 +1425,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, @@ -1581,23 +1444,17 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real * const cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex; #if !defined(WARPX_DIM_RZ) ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -1628,21 +1485,9 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif #if !defined(WARPX_DIM_3D) constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); @@ -2190,13 +2035,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -2224,9 +2062,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ * \param[in] lo Dimension-agnostic lower bounds of index domain * \param[in] q Species charge * \param[in] n_rz_azimuthal_modes Number of azimuthal modes in RZ geometry - * \param[in,out] cost Pointer to (load balancing) cost corresponding to box where - present particles deposit current - * \param[in] load_balance_costs_update_algo Selected method for updating load balance costs */ template void doVayDepositionShapeN (const GetParticlePosition& GetPosition, @@ -2245,9 +2080,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, const std::array& xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex::literals; @@ -2265,10 +2098,6 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); #endif -#if !defined(WARPX_USE_GPUCLOCK) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) amrex::ignore_unused(n_rz_azimuthal_modes); @@ -2319,21 +2148,8 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Array4 const& Dz_arr = Dz_fab.array(); // Loop over particles and deposit (Dx,Dy,Dz) into Dx_fab, Dy_fab and Dz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor(np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - // Inverse of Lorentz factor gamma const amrex::Real invgam = 1._rt / std::sqrt(1._rt + uxp[ip] * uxp[ip] * invcsq + uyp[ip] * uyp[ip] * invcsq @@ -2560,14 +2376,6 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, // Synchronize so that temp_fab can be safely deallocated in its destructor amrex::Gpu::streamSynchronize(); - -# if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -# endif #endif // #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) } #endif // CURRENTDEPOSITION_H_ diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 46101c29e0b..fd1bc7917ab 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -461,8 +461,6 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, } WARPX_PROFILE_VAR_START(blp_deposit); - amrex::LayoutData * const costs = WarpX::getCosts(lev); - amrex::Real * const cost = costs ? &((*costs)[pti.index()]) : nullptr; // If doing shared mem current deposition, get tile info if (WarpX::do_shared_mem_current_deposition) { @@ -531,29 +529,29 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 4){ doDepositionSharedShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -567,29 +565,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) @@ -616,8 +610,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doChargeConservingDepositionShapeNImplicit<2>( xp_n_data, yp_n_data, zp_n_data, @@ -625,8 +618,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doChargeConservingDepositionShapeNImplicit<3>( xp_n_data, yp_n_data, zp_n_data, @@ -634,8 +626,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doChargeConservingDepositionShapeNImplicit<4>( xp_n_data, yp_n_data, zp_n_data, @@ -643,8 +634,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -673,8 +663,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVillasenorDepositionShapeNImplicit<2>( xp_n_data, yp_n_data, zp_n_data, @@ -682,8 +671,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVillasenorDepositionShapeNImplicit<3>( xp_n_data, yp_n_data, zp_n_data, @@ -691,8 +679,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVillasenorDepositionShapeNImplicit<4>( xp_n_data, yp_n_data, zp_n_data, @@ -700,8 +687,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else { @@ -716,29 +702,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVayDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else { // Direct deposition if (push_type == PushType::Explicit) { @@ -747,29 +729,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); @@ -782,8 +760,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeNImplicit<2>( GetPosition, wp.dataPtr() + offset, @@ -791,8 +768,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeNImplicit<3>( GetPosition, wp.dataPtr() + offset, @@ -800,8 +776,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeNImplicit<4>( GetPosition, wp.dataPtr() + offset, @@ -809,8 +784,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } } @@ -1084,8 +1058,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } WARPX_PROFILE_VAR_START(blp_ppc_chd); - amrex::LayoutData* costs = WarpX::getCosts(lev); - amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; const auto GetPosition = GetParticlePosition(pti, offset); const Geometry& geom = Geom(lev); @@ -1095,26 +1067,26 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 4){ doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } #ifndef AMREX_USE_GPU @@ -1158,10 +1130,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); - // pointer to costs data - amrex::LayoutData* costs = WarpX::getCosts(lev); - amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; - AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noy); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); @@ -1172,7 +1140,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, icomp, nc, - cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize + WarpX::do_device_synchronize ); } } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index e94b7bb7719..b7af3121814 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -132,9 +132,8 @@ struct RhoInTime { struct LoadBalanceCostsUpdateAlgo { enum { Timers = 0, //!< load balance according to in-code timer-based weights (i.e., with `costs`) - Heuristic = 1, /**< load balance according to weights computed from number of cells - and number of particles per box (i.e., with `costs_heuristic`)*/ - GpuClock = 2 + Heuristic = 1 /**< load balance according to weights computed from number of cells + and number of particles per box (i.e., with `costs_heuristic`) */ }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index c58318bbd53..3092e1984d6 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -101,7 +101,6 @@ const std::map rho_in_time_to_int = { const std::map load_balance_costs_update_algo_to_int = { {"timers", LoadBalanceCostsUpdateAlgo::Timers }, - {"gpuclock", LoadBalanceCostsUpdateAlgo::GpuClock }, {"heuristic", LoadBalanceCostsUpdateAlgo::Heuristic }, {"default", LoadBalanceCostsUpdateAlgo::Timers } }; diff --git a/Source/WarpX.H b/Source/WarpX.H index d7b7266d7b0..76b9b99e9a1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -176,7 +176,7 @@ public: //! Flags whether the Picard iterations are required to converge static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy - * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). + * being used (0 or 1 corresponding to timers or heuristic). */ static short load_balance_costs_update_algo; //! Integer that corresponds to electromagnetic Maxwell solver (vacuum - 0, macroscopic - 1) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c929718b043..5caeb9ffb2e 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1320,10 +1320,6 @@ WarpX::ReadParameters () utils::parser::queryWithParser( pp_algo, "costs_heuristic_particles_wt", costs_heuristic_particles_wt); } -# ifndef WARPX_USE_GPUCLOCK - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(WarpX::load_balance_costs_update_algo!=LoadBalanceCostsUpdateAlgo::GpuClock, - "`algo.load_balance_costs_update = gpuclock` requires to compile with `-DWarpX_GPUCLOCK=ON`."); -# endif // !WARPX_USE_GPUCLOCK // Parse algo.particle_shape and check that input is acceptable // (do this only if there is at least one particle or laser species) diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index f43e35c6b0b..4d78e309f85 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -44,9 +44,6 @@ namespace ablastr::particles * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit - * \param cost pointer to (load balancing) cost corresponding to box where present - particles deposit current. If nullptr, costs are not updated. (default: nullptr) - * \param load_balance_costs_update_algo selected method for updating load balance costs (default: 0) * \param do_device_synchronize call amrex::Gpu::synchronize() for tiny profiler regions (default: true) */ template< typename T_PC > @@ -67,8 +64,6 @@ deposit_charge (typename T_PC::ParIterType& pti, long const offset = 0, std::optional np_to_deposit = std::nullopt, int const icomp = 0, int const nc = 1, - amrex::Real * const AMREX_RESTRICT cost = nullptr, - long const load_balance_costs_update_algo = 0, bool const do_device_synchronize = true) { // deposition guards @@ -183,23 +178,19 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 4){ doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); diff --git a/Tools/PerformanceTests/summit.py b/Tools/PerformanceTests/summit.py index cb0200e197c..c2ba6c70a2e 100644 --- a/Tools/PerformanceTests/summit.py +++ b/Tools/PerformanceTests/summit.py @@ -15,7 +15,7 @@ def executable_name(compiler,architecture): - return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.GPUCLOCK.ex' + return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.ex' def get_config_command(compiler, architecture): config_command = '' diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 03225878700..5a2c0916432 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -17,7 +17,6 @@ set(WarpX_EB @WarpX_EB@) set(WarpX_EB_FOUND ${WarpX_EB}) set(WarpX_LIB @WarpX_LIB@) set(WarpX_LIB_FOUND ${WarpX_LIB}) -WarpX_GPUCLOCK # dependencies set(WarpX_MPI @WarpX_MPI@) diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 6e0e2ca5d1d..9b9feb64437 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -444,7 +444,6 @@ function(warpx_print_summary) message(" COMPUTE: ${WarpX_COMPUTE}") message(" DIMS: ${WarpX_DIMS}") message(" Embedded Boundary: ${WarpX_EB}") - message(" GPU clock timers: ${WarpX_GPUCLOCK}") message(" IPO/LTO: ${WarpX_IPO}") message(" LIB: ${WarpX_LIB}${LIB_TYPE}") message(" MPI: ${WarpX_MPI}") From 5857c0462238b03b76bb8ab5126bda689d1a9b2d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 26 Mar 2024 03:30:31 -0700 Subject: [PATCH 031/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4803) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index feb779d6ad4..739684f6c74 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - + cd ../amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index d5ddbe2614d..8f7da493f24 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 +branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 69b9f0d4b4d..4ac74d14310 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 +branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 4c9a7bcb906..fdc628f3f06 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "a8633589c1fae5b3fda6deca86fba79878a8cb54" +set(WarpX_amrex_branch "9e8f8d60eca337e3c14396f06f0a77500289d45c" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 3242747b6f5..e2b83469010 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "e028359d5679e0775b4a8a36332c5d1d37fc7506" +set(WarpX_pyamrex_branch "bde190184effaf095f14e173df8f61f31ed67308" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 081efe21626..593d7bf07ca 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - +cd amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 6b0a040d1289f31899c67c2182e4f558a8a8d9ba Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 26 Mar 2024 11:35:50 -0700 Subject: [PATCH 032/121] Cleaning: `.rst` Ending in TOC (#4804) Follow-up to #3137 --- Docs/source/usage/workflows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index b68810a4345..039be199257 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -12,7 +12,7 @@ This section collects typical user workflows and best practices for WarpX. workflows/domain_decomposition workflows/plot_distribution_mapping workflows/debugging - workflows/generate_lookup_tables_with_tools.rst + workflows/generate_lookup_tables_with_tools workflows/libensemble workflows/plot_timestep_duration workflows/psatd_stencil From fdbf455f8a47ce0f160a9848551d2ab8f96ab5b3 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 28 Mar 2024 18:32:31 +0100 Subject: [PATCH 033/121] update fugaku_warpx.profile (#4808) --- Tools/machines/fugaku-riken/fugaku_warpx.profile.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example index a430579a7d5..caf85983b58 100644 --- a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example +++ b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example @@ -10,7 +10,7 @@ export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH spack load ninja@1.11.1%fj@4.10.0 # optional: for PSATD -spack load fujitsu-fftw +spack load fujitsu-fftw@1.1.0%fj@4.10.0 # optional: for QED lookup table generation support spack load boost@1.80.0%fj@4.8.1/zc5pwgc From 697fc5b37ee65d00483f968899e57a3049e442e6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Mar 2024 13:09:06 -0700 Subject: [PATCH 034/121] Evolve Loop: Simplify (#4791) * Evolve Loop: Simplify Introduce structure functions to simplify the number of calls in our main evolve loop. * Code Review (Dave) Thank you! :) * Update Timer Cost Rescaling * simplify return logic Co-authored-by: Luca Fedeli * Rename: `HandleParticlesAtBoundaries` from `InjectRedistributeScapeParticles` * Fix Typo: scrape --------- Co-authored-by: Luca Fedeli Co-authored-by: David Grote --- Source/Evolve/WarpXEvolve.cpp | 278 +++++++++++++------------ Source/Parallelization/WarpXRegrid.cpp | 47 +++++ Source/WarpX.H | 46 ++++ 3 files changed, 237 insertions(+), 134 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index dd6acfd2694..a8aa89355ed 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -70,8 +70,8 @@ WarpX::Evolve (int numsteps) // Note that the default argument is numsteps = -1 const int numsteps_max = (numsteps < 0)?(max_step):(istep[0] + numsteps); - bool early_params_checked = false; // check typos in inputs after step 1 - bool exit_loop_due_to_interrupt_signal = false; + // check typos in inputs after step 1 + bool early_params_checked = false; static Real evolve_time = 0; @@ -81,7 +81,7 @@ WarpX::Evolve (int numsteps) WARPX_PROFILE("WarpX::Evolve::step"); const auto evolve_time_beg_step = static_cast(amrex::second()); - //Check and clear signal flags and asynchronously broadcast them from process 0 + // Check and clear signal flags and asynchronously broadcast them from process 0 SignalHandling::CheckSignals(); multi_diags->NewIteration(); @@ -92,90 +92,29 @@ WarpX::Evolve (int numsteps) } ExecutePythonCallback("beforestep"); - amrex::LayoutData* cost = WarpX::getCosts(0); - if (cost) { - if (step > 0 && load_balance_intervals.contains(step+1)) - { - LoadBalance(); + CheckLoadBalance(step); - // Reset the costs to 0 - ResetCosts(); - } - for (int lev = 0; lev <= finest_level; ++lev) - { - cost = WarpX::getCosts(lev); - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - // Perform running average of the costs - // (Giving more importance to most recent costs; only needed - // for timers update, heuristic load balance considers the - // instantaneous costs) - for (const auto& i : cost->IndexArray()) - { - (*cost)[i] *= (1._rt - 2._rt/load_balance_intervals.localPeriod(step+1)); - } - } - } - } - - if (evolve_scheme == EvolveScheme::Explicit) { - // At the beginning, we have B^{n} and E^{n}. - // Particles have p^{n} and x^{n}. - // is_synchronized is true. - - if (is_synchronized) { - // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ng_alloc_EB); - FillBoundaryB(guard_cells.ng_alloc_EB); - - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - // on first step, push p by -0.5*dt - for (int lev = 0; lev <= finest_level; ++lev) - { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - - } else { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); - } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - FillBoundaryAux(guard_cells.ng_UpdateAux); - } - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - } + if (evolve_scheme == EvolveScheme::Explicit) + { + ExplicitFillBoundaryEBUpdateAux(); } // If needed, deposit the initial ion charge and current densities that // will be used to update the E-field in Ohm's law. if (step == step_begin && electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC - ) { HybridPICDepositInitialRhoAndJ(); } + ) { + HybridPICDepositInitialRhoAndJ(); + } // Run multi-physics modules: // ionization, Coulomb collisions, QED doFieldIonization(); + ExecutePythonCallback("beforecollisions"); mypc->doCollisions( cur_time, dt[0] ); ExecutePythonCallback("aftercollisions"); + #ifdef WARPX_QED doQEDEvents(); mypc->doQEDSchwinger(); @@ -186,6 +125,7 @@ WarpX::Evolve (int numsteps) ExecutePythonCallback("particleinjection"); + // TODO: move out if (evolve_scheme == EvolveScheme::ImplicitPicard || evolve_scheme == EvolveScheme::SemiImplicitPicard) { OneStep_ImplicitPicard(cur_time); @@ -229,12 +169,13 @@ WarpX::Evolve (int numsteps) mypc->doResampling(istep[0]+1, verbose); if (evolve_scheme == EvolveScheme::Explicit) { - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date + } + // TODO: move out + if (evolve_scheme == EvolveScheme::Explicit) { if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { // At the end of last step, push p by 0.5*dt to synchronize FillBoundaryE(guard_cells.ng_FieldGather); @@ -284,49 +225,7 @@ WarpX::Evolve (int numsteps) } } - mypc->ContinuousFluxInjection(cur_time, dt[0]); - - mypc->ApplyBoundaryConditions(); - - // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); -#endif - - m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); - - // Non-Maxwell solver: particles can move by an arbitrary number of cells - if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || - electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) - { - mypc->Redistribute(); - } - else - { - // Electromagnetic solver: due to CFL condition, particles can - // only move by one or two cells per time step - if (max_level == 0) { - int num_redistribute_ghost = num_moved; - if ((m_v_galilean[0]!=0) or (m_v_galilean[1]!=0) or (m_v_galilean[2]!=0)) { - // Galilean algorithm ; particles can move by up to 2 cells - num_redistribute_ghost += 2; - } else { - // Standard algorithm ; particles can move by up to 1 cell - num_redistribute_ghost += 1; - } - mypc->RedistributeLocal(num_redistribute_ghost); - } - else { - mypc->Redistribute(); - } - } - - if (sort_intervals.contains(step+1)) { - if (verbose) { - amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); - } - mypc->SortParticlesByBin(sort_bin_size); - } + HandleParticlesAtBoundaries(step, cur_time, num_moved); // Field solve step for electrostatic or hybrid-PIC solvers if( electrostatic_solver_id != ElectrostaticSolverAlgo::None || @@ -378,12 +277,7 @@ WarpX::Evolve (int numsteps) // inputs: unused parameters (e.g. typos) check after step 1 has finished if (!early_params_checked) { - amrex::Print() << "\n"; // better: conditional \n based on return value - amrex::ParmParse::QueryUnusedInputs(); - - //Print the warning list right after the first step. - amrex::Print() << - ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); + checkEarlyUnusedParams(); early_params_checked = true; } @@ -401,20 +295,18 @@ WarpX::Evolve (int numsteps) << " s; Avg. per step = " << evolve_time/(step-step_begin+1) << " s\n\n"; } - exit_loop_due_to_interrupt_signal = SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK); - if (cur_time >= stop_time - 1.e-3*dt[0] || exit_loop_due_to_interrupt_signal) { + if (checkStopSimulation(cur_time)) { break; } + } // End loop on time steps - // End loop on time steps - } // This if statement is needed for PICMI, which allows the Evolve routine to be // called multiple times, otherwise diagnostics will be done at every call, // regardless of the diagnostic period parameter provided in the inputs. if (istep[0] == max_step || (stop_time - 1.e-3*dt[0] <= cur_time && cur_time < stop_time + dt[0]) - || exit_loop_due_to_interrupt_signal) { + || m_exit_loop_due_to_interrupt_signal) { multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } + if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -526,6 +418,119 @@ WarpX::OneStep_nosub (Real cur_time) ExecutePythonCallback("afterEsolve"); } +bool WarpX::checkStopSimulation (amrex::Real cur_time) +{ + m_exit_loop_due_to_interrupt_signal = SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK); + return (cur_time >= stop_time - 1.e-3*dt[0]) || + m_exit_loop_due_to_interrupt_signal; +} + +void WarpX::checkEarlyUnusedParams () +{ + amrex::Print() << "\n"; // better: conditional \n based on return value + amrex::ParmParse::QueryUnusedInputs(); + + // Print the warning list right after the first step. + amrex::Print() << ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); +} + +void WarpX::ExplicitFillBoundaryEBUpdateAux () +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(evolve_scheme == EvolveScheme::Explicit, + "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux wihtout Explicit evolve scheme set!"); + + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + + if (is_synchronized) { + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) + { + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + } + is_synchronized = false; + + } else { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + } +} + +void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved) +{ + mypc->ContinuousFluxInjection(cur_time, dt[0]); + + mypc->ApplyBoundaryConditions(); + + // interact the particles with EB walls (if present) +#ifdef AMREX_USE_EB + mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); +#endif + + m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + + // Non-Maxwell solver: particles can move by an arbitrary number of cells + if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) + { + mypc->Redistribute(); + } + else + { + // Electromagnetic solver: due to CFL condition, particles can + // only move by one or two cells per time step + if (max_level == 0) { + int num_redistribute_ghost = num_moved; + if ((m_v_galilean[0]!=0) or (m_v_galilean[1]!=0) or (m_v_galilean[2]!=0)) { + // Galilean algorithm ; particles can move by up to 2 cells + num_redistribute_ghost += 2; + } else { + // Standard algorithm ; particles can move by up to 1 cell + num_redistribute_ghost += 1; + } + mypc->RedistributeLocal(num_redistribute_ghost); + } + else { + mypc->Redistribute(); + } + } + + if (sort_intervals.contains(step+1)) { + if (verbose) { + amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); + } + mypc->SortParticlesByBin(sort_bin_size); + } +} + void WarpX::SyncCurrentAndRho () { if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -1054,8 +1059,13 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, * The mirror normal direction has to be parallel to the z axis. */ void -WarpX::applyMirrors(Real time) +WarpX::applyMirrors (Real time) { + // something to do? + if (num_mirrors == 0) { + return; + } + // Loop over the mirrors for(int i_mirror=0; i_mirror 0 && load_balance_intervals.contains(step+1)) + { + LoadBalance(); + + // Reset the costs to 0 + ResetCosts(); + } + if (!costs.empty()) + { + RescaleCosts(step); + } +} + void WarpX::LoadBalance () { WARPX_PROFILE_REGION("LoadBalance"); WARPX_PROFILE("WarpX::LoadBalance()"); + AMREX_ALWAYS_ASSERT(!costs.empty()); AMREX_ALWAYS_ASSERT(costs[0] != nullptr); #ifdef AMREX_USE_MPI @@ -401,6 +418,9 @@ WarpX::ComputeCostsHeuristic (amrex::VectorIndexArray(); @@ -411,3 +431,30 @@ WarpX::ResetCosts () } } } + +void +WarpX::RescaleCosts (int step) +{ + // rescale is only used for timers + if (WarpX::load_balance_costs_update_algo != LoadBalanceCostsUpdateAlgo::Timers) + { + return; + } + + AMREX_ALWAYS_ASSERT(costs.size() == finest_level + 1); + + for (int lev = 0; lev <= finest_level; ++lev) + { + if (costs[lev]) + { + // Perform running average of the costs + // (Giving more importance to most recent costs; only needed + // for timers update, heuristic load balance considers the + // instantaneous costs) + for (const auto& i : costs[lev]->IndexArray()) + { + (*costs[lev])[i] *= (1._rt - 2._rt/load_balance_intervals.localPeriod(step+1)); + } + } + } +} diff --git a/Source/WarpX.H b/Source/WarpX.H index 76b9b99e9a1..d9ca494e72b 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -664,13 +664,26 @@ public: static amrex::Real quantum_xi_c2; + /** Check and potentially compute load balancing + */ + void CheckLoadBalance (int step); + /** \brief perform load balance; compute and communicate new `amrex::DistributionMapping` */ void LoadBalance (); + /** \brief resets costs to zero */ void ResetCosts (); + /** Perform running average of the LB costs + * + * Only needed for timers cost update, heuristic load balance considers the + * instantaneous costs. + * This gives more importance to most recent costs. + */ + void RescaleCosts (int step); + /** \brief returns the load balance interval */ [[nodiscard]] utils::parser::IntervalsParser get_load_balance_intervals () const @@ -1723,8 +1736,41 @@ private: return *m_field_factory[lev]; } + /** Stop the simulation at the end of the current step due to a received Unix signal? + */ + bool m_exit_loop_due_to_interrupt_signal = false; + + /** Stop the simulation at the end of the current step? + */ + [[nodiscard]] + bool checkStopSimulation (amrex::Real cur_time); + + /** Print Unused Parameter Warnings after Step 1 + * + * Instead of waiting for a simulation to end, we already do an early "unused parameter check" + * after step 1 to inform users early of potential issues with their simulation setup. + */ + void checkEarlyUnusedParams (); + + /** Perform essential particle house keeping at boundaries + * + * Inject, communicate, scrape and sort particles. + * + * @param step current step + * @param cur_time current time + * @param num_moved number of cells the moving window moved + */ + void HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved); + void ScrapeParticles (); + /** Update the E and B fields in the explicit em PIC scheme. + * + * At the beginning, we have B^{n} and E^{n}. + * Particles have p^{n} and x^{n}. + */ + void ExplicitFillBoundaryEBUpdateAux (); + void PushPSATD (); #ifdef WARPX_USE_PSATD From 5cfa8f13f32c2a3c678e33ec230395777af49df0 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 29 Mar 2024 19:34:16 +0100 Subject: [PATCH 035/121] fix typos in scripts to run WarpX on Frontier (#4811) --- Tools/machines/frontier-olcf/install_dependencies.sh | 2 +- Tools/machines/frontier-olcf/submit.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 9460f9a5175..98c30cfca8f 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -14,7 +14,7 @@ set -eu -o pipefail # Check: ###################################################################### # -# Was perlmutter_gpu_warpx.profile sourced and configured correctly? +# Was frontier_warpx.profile sourced and configured correctly? if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your frontier_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi diff --git a/Tools/machines/frontier-olcf/submit.sh b/Tools/machines/frontier-olcf/submit.sh index 13d981238a3..b340d56546e 100644 --- a/Tools/machines/frontier-olcf/submit.sh +++ b/Tools/machines/frontier-olcf/submit.sh @@ -32,7 +32,7 @@ export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # Seen since August 2023 -# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors # https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors export MPICH_SMP_SINGLE_COPY_MODE=NONE export FI_CXI_RX_MATCH_MODE=software From 72800eab2b889bb67b5c90f14929b679c715c34c Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Sat, 30 Mar 2024 19:26:03 -0700 Subject: [PATCH 036/121] Add thermalBC for particles (#4790) * add thermal as particle boundary type * separate file for gaussian distribution function * parse and apply thermal boundary * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix typo * fixing unused var * isAnyThermal as static fctn is warpx class, instead of particle boundaries struct * unused var for 1D, 2D, RZ * change API for u_th * add thermal for PEC * file name change * Update Source/Particles/ParticleBoundaries_K.H Co-authored-by: David Grote * restructure rethermalize within ifdef for apply boundary * add CI test * doc * fix py file name * condition for 0 uth * typo * fix path for reduceddiags * no chksum too noisy * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * addchksum * import os * Update Examples/Tests/particle_thermal_boundary/analysis_2d.py * Update Source/Particles/ParticleBoundaries_K.H --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: David Grote Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 6 ++ .../particle_thermal_boundary/analysis_2d.py | 37 +++++++++ .../Tests/particle_thermal_boundary/inputs_2d | 77 ++++++++++++++++++ .../particle_thermal_boundary.json | 12 +++ Regression/WarpX-tests.ini | 17 ++++ .../WarpXFieldBoundaries.cpp | 6 +- Source/BoundaryConditions/WarpX_PEC.cpp | 22 +++-- Source/Initialization/InjectorMomentum.H | 70 +--------------- .../SampleGaussianFluxDistribution.H | 81 +++++++++++++++++++ Source/Particles/ParticleBoundaries.H | 3 + Source/Particles/ParticleBoundaries.cpp | 7 ++ Source/Particles/ParticleBoundaries_K.H | 53 ++++++++++-- .../Particles/PhysicalParticleContainer.cpp | 8 ++ Source/Utils/WarpXAlgorithmSelection.H | 3 +- Source/Utils/WarpXAlgorithmSelection.cpp | 1 + Source/WarpX.H | 6 +- Source/WarpX.cpp | 10 +++ 17 files changed, 332 insertions(+), 87 deletions(-) create mode 100755 Examples/Tests/particle_thermal_boundary/analysis_2d.py create mode 100644 Examples/Tests/particle_thermal_boundary/inputs_2d create mode 100644 Regression/Checksum/benchmarks_json/particle_thermal_boundary.json create mode 100644 Source/Initialization/SampleGaussianFluxDistribution.H diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index fc6d338fc1c..fa1d5e99670 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -417,6 +417,12 @@ Domain Boundary Conditions * ``Reflecting``: Particles leaving the boundary are reflected from the boundary back into the domain. When ``boundary.reflect_all_velocities`` is false, the sign of only the normal velocity is changed, otherwise the sign of all velocities are changed. + * ``Thermal``: Particles leaving the boundary are reflected from the boundary back into the domain + and their velocities are thermalized. The tangential velocity components are sampled from ``gaussian`` distribution + and the component normal to the boundary is sampled from ``gaussian flux`` distribution. + The standard deviation for these distributions should be provided for each species using + ``boundary..u_th``. The same standard deviation is used to sample all components. + * ``boundary.reflect_all_velocities`` (`bool`) optional (default `false`) For a reflecting boundary condition, this flags whether the sign of only the normal velocity is changed or all velocities. diff --git a/Examples/Tests/particle_thermal_boundary/analysis_2d.py b/Examples/Tests/particle_thermal_boundary/analysis_2d.py new file mode 100755 index 00000000000..db14479af2c --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/analysis_2d.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# +# Copyright 2023-2024 Revathi Jambunathan +# +# This file is part of WarpX +# +# License: BSD-3-Clause-LBNL + +""" +The script checks to see the growth in total field energy and total particle energy + +The input file in 2D initializes uniform plasma (electrons,ions) with +thermal boundary condition. We do not expect the particle energy to increase +beyond 2% in the time that it takes all particles to cross the domain boundary +""" + +import os +import sys + +import numpy as np + +sys.path.insert(1,'../../../../warpx/Regression/Checksum/') +import checksumAPI + +FE_rdiag = './diags/reducedfiles/EF.txt' +init_Fenergy = np.loadtxt(FE_rdiag)[1,2] +final_Fenergy = np.loadtxt(FE_rdiag)[-1,2] +assert(final_Fenergy/init_Fenergy < 40) +assert(final_Fenergy < 5.e-5) + +PE_rdiag = './diags/reducedfiles/EN.txt' +init_Penergy = np.loadtxt(PE_rdiag)[0,2] +final_Penergy = np.loadtxt(PE_rdiag)[-1,2] +assert( abs(final_Penergy - init_Penergy)/init_Penergy < 0.02) +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particle_thermal_boundary/inputs_2d b/Examples/Tests/particle_thermal_boundary/inputs_2d new file mode 100644 index 00000000000..9c3e18cd35d --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/inputs_2d @@ -0,0 +1,77 @@ +max_step = 2000 + +# number of grid points +amr.n_cell = 16 16 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 128 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = 0.e-6 0.e-6 # physical domain +geometry.prob_hi = 2.5e-7 2.5e-7 + +# Boundary condition +boundary.field_lo = pml pml +boundary.field_hi = pml pml +boundary.particle_lo = thermal thermal +boundary.particle_hi = thermal thermal +boundary.electrons.u_th = uth_e +boundary.C.u_th = uth_C +warpx.do_dive_cleaning = 0 + +# Verbosity +warpx.verbose = 1 + +# Order of particle shape factors +algo.particle_shape = 2 + +# CFL +warpx.cfl = 0.98 + +# Density +my_constants.n0 = 4e26 +my_constants.uth_e = 0.06256112470898544 +my_constants.uth_C = 0.00042148059678527106 + +# Particles +particles.species_names = electrons C + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = uth_e +electrons.uy_th = uth_e +electrons.uz_th = uth_e + +C.charge = 6*q_e +C.mass = 12*m_p +C.injection_style = "NUniformPerCell" +C.num_particles_per_cell_each_dim = 8 8 +C.profile = constant +C.density = n0/6 +C.momentum_distribution_type = gaussian +C.ux_th = uth_C +C.uy_th = uth_C +C.uz_th = uth_C + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 3000 +diag1.write_species = 0 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE +diag1.diag_type = Full + +warpx.reduced_diags_names = EN EF +EN.intervals = 10 +EN.type = ParticleEnergy +EF.intervals = 10 +EF.type = FieldEnergy diff --git a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json new file mode 100644 index 00000000000..5c302d4c5b3 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json @@ -0,0 +1,12 @@ +{ + "lev=0": { + "Bx": 167.49421217759414, + "By": 269.6793406439766, + "Bz": 258.3282607081799, + "Ex": 864657516595.9229, + "Ey": 24072965021.088013, + "Ez": 702786324880.7338, + "divE": 5.137699003981513e+19, + "rho": 357057125.6699786 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 4ac74d14310..42274fa30ae 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4720,3 +4720,20 @@ compareParticles = 1 particleTypes = electrons outputFile = particle_boundary_interaction_plt analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py + +[particle_thermal_boundary] +buildDir = . +inputFile = Examples/Tests/particle_thermal_boundary/inputs_2d +runtime_params = +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index a9e2cf29fe9..4844585cc90 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -106,7 +106,8 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || PEC::isAnyBoundaryPEC()) { + if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + { PEC::ApplyReflectiveBoundarytoRhofield(rho, lev, patch_type); } } @@ -115,7 +116,8 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || PEC::isAnyBoundaryPEC()) { + if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + { PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, lev, patch_type); } } diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 0b4b4dae30a..a3d92fa507d 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -258,8 +258,10 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -268,9 +270,11 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat // components of the current density is_tangent_to_bndy[idim] = true; - psign[idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[idim][0] = ( ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; - psign[idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - rho_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - rho_nodal[idim]); @@ -352,8 +356,10 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -375,15 +381,19 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, #endif if (is_tangent_to_bndy[icomp][idim]){ - psign[icomp][idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; - psign[icomp][idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; } else { - psign[icomp][idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) ? -1._rt : 1._rt; - psign[icomp][idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) ? -1._rt : 1._rt; } } diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index f7ee6cff138..2b8b9065a36 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -12,6 +12,7 @@ #include "GetVelocity.H" #include "TemperatureProperties.H" #include "VelocityProperties.H" +#include "SampleGaussianFluxDistribution.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -90,75 +91,6 @@ private: amrex::Real m_ux_th, m_uy_th, m_uz_th; }; -namespace { - /** Return u sampled according to the probability distribution: - * p(u) \propto u \exp(-(u-u_m)^2/2u_th^2) - * - * @param u_m Central momentum - * @param u_th Momentum spread - * @param engine Object used to generate random numbers - */ - [[nodiscard]] - AMREX_FORCE_INLINE - AMREX_GPU_HOST_DEVICE - amrex::Real - generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { - - using namespace amrex::literals; - - // Momentum to be returned at the end of this function - amrex::Real u = 0._rt; - - const amrex::Real abs_u_m = std::abs(u_m); - - if (u_th == 0._rt) { - u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below - } else if (abs_u_m < 0.6*u_th) { - // Mean velocity magnitude is less than thermal velocity - // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation - // and then use the rejection method to correct it - // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) - // Note that this is the method that is used in the common case u_m=0 - const amrex::Real umsign = std::copysign(1._rt, u_m); - const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); - const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation - bool reject = true; - while (reject) { - // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), - // using the method of the inverse cumulative function - amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 - u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); - // Rejection method - xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } - } - } else { - // Mean velocity magnitude is greater than thermal velocity - // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation - // and then use the rejection method to correct it - // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note - // that this number is always between 0 and 1 ) - // Note that in the common case `u_m = 0`, this rejection method - // is not used, and the above rejection method is used instead. - bool reject = true; - const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; - const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation - while (reject) { - // Approximate distribution: normal distribution, where we only retain positive u - u = -1._rt; - while (u < 0) { - u = amrex::RandomNormal(approx_u_m, u_th, engine); - } - // Rejection method - const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } - } - } - - return u; - } -} - // struct whose getMomentum returns momentum for 1 particle, from random // gaussian flux distribution in the specified direction. // Along the normal axis, the distribution is v*Gaussian, diff --git a/Source/Initialization/SampleGaussianFluxDistribution.H b/Source/Initialization/SampleGaussianFluxDistribution.H new file mode 100644 index 00000000000..407c9df632a --- /dev/null +++ b/Source/Initialization/SampleGaussianFluxDistribution.H @@ -0,0 +1,81 @@ +/* Copyright 2024 Remi Lehe, Revathi Jambunathan + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#define SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H + +#include + +namespace { + /** This function returns u sampled according to the probability distribution: + * p(u) \propto u \exp(-(u-u_m)^2/2u_th^2) + * + * @param u_m Central momentum + * @param u_th Momentum spread + * @param engine Object used to generate random numbers + */ + [[nodiscard]] + AMREX_FORCE_INLINE + AMREX_GPU_HOST_DEVICE + amrex::Real + generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { + + using namespace amrex::literals; + + // Momentum to be returned at the end of this function + amrex::Real u = 0._rt; + + const amrex::Real abs_u_m = std::abs(u_m); + + if (u_th == 0._rt) { + u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below + } else if (abs_u_m < 0.6*u_th) { + // Mean velocity magnitude is less than thermal velocity + // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation + // and then use the rejection method to correct it + // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) + // Note that this is the method that is used in the common case u_m=0 + const amrex::Real umsign = std::copysign(1._rt, u_m); + const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); + const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation + bool reject = true; + while (reject) { + // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), + // using the method of the inverse cumulative function + amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 + u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); + // Rejection method + xrand = amrex::Random(engine); + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } + } + } else { + // Mean velocity magnitude is greater than thermal velocity + // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation + // and then use the rejection method to correct it + // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note + // that this number is always between 0 and 1 ) + // Note that in the common case `u_m = 0`, this rejection method + // is not used, and the above rejection method is used instead. + bool reject = true; + const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; + const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation + while (reject) { + // Approximate distribution: normal distribution, where we only retain positive u + u = -1._rt; + while (u < 0) { + u = amrex::RandomNormal(approx_u_m, u_th, engine); + } + // Rejection method + const amrex::Real xrand = amrex::Random(engine); + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } + } + } + return u; + } +} + +#endif diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index 78b090056b1..c056436770d 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -22,6 +22,8 @@ struct ParticleBoundaries void Set_reflect_all_velocities (bool flag); void SetAll (ParticleBoundaryType bc); + /** Sets thermal velocity in ParticleBoundariesData 'data.m_uth' to u_th */ + void SetThermalVelocity (amrex::Real u_th); void SetBoundsX (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); void SetBoundsY (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); @@ -54,6 +56,7 @@ struct ParticleBoundaries ParticleBoundaryType ymax_bc; ParticleBoundaryType zmin_bc; ParticleBoundaryType zmax_bc; + amrex::Real m_uth = 0.; amrex::ParserExecutor<1> reflection_model_xlo; amrex::ParserExecutor<1> reflection_model_xhi; diff --git a/Source/Particles/ParticleBoundaries.cpp b/Source/Particles/ParticleBoundaries.cpp index 5b51fa3fd25..10268a9c01f 100644 --- a/Source/Particles/ParticleBoundaries.cpp +++ b/Source/Particles/ParticleBoundaries.cpp @@ -6,6 +6,7 @@ */ #include "ParticleBoundaries.H" +#include "WarpX.H" #include "Utils/Parser/ParserUtils.H" @@ -32,6 +33,12 @@ ParticleBoundaries::SetAll (ParticleBoundaryType bc) data.zmax_bc = bc; } +void +ParticleBoundaries::SetThermalVelocity (amrex::Real u_th) +{ + data.m_uth = u_th; +} + void ParticleBoundaries::SetBoundsX (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi) { diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index bbec1f54e01..1481dca7684 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -1,4 +1,4 @@ -/* Copyright 2021 David Grote +/* Copyright 2021 David Grote, Remi Lehe, Revathi Jambunathan * * This file is part of WarpX. * @@ -8,18 +8,19 @@ #define PARTICLEBOUNDARIES_K_H_ #include "ParticleBoundaries.H" +#include "Initialization/SampleGaussianFluxDistribution.H" #include namespace ApplyParticleBoundaries { - + using namespace amrex::literals; /* \brief Applies the boundary condition on a specific axis * This is called by apply_boundaries. */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void apply_boundary (amrex::ParticleReal& x, amrex::Real xmin, amrex::Real xmax, - bool& change_sign_ux, bool& particle_lost, + bool& change_sign_ux, bool& rethermalize_x, bool& particle_lost, ParticleBoundaryType xmin_bc, ParticleBoundaryType xmax_bc, amrex::Real refl_probability_xmin, amrex::Real refl_probability_xmax, amrex::RandomEngine const& engine ) @@ -42,6 +43,10 @@ namespace ApplyParticleBoundaries { x = 2*xmin - x; change_sign_ux = true; } + else if (xmin_bc == ParticleBoundaryType::Thermal) { + x = 2*xmin - x; + rethermalize_x = true; + } } else if (x > xmax) { if (xmax_bc == ParticleBoundaryType::Open) { @@ -61,16 +66,38 @@ namespace ApplyParticleBoundaries { x = 2*xmax - x; change_sign_ux = true; } + else if (xmax_bc == ParticleBoundaryType::Thermal) { + x = 2*xmax - x; + rethermalize_x = true; + } } } - /* \brief Applies absorbing or reflecting boundary condition to the input particles, along all axis. + /* \brief Thermalize particles that have been identified to cross the boundary. + * The normal component samples from a half-Maxwellian, + * and the two tangential components will sample from full Maxwellian disbutions + * with thermal velocity uth + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void thermalize_boundary_particle (amrex::ParticleReal& u_norm, amrex::ParticleReal& u_tang1, + amrex::ParticleReal& u_tang2, amrex::Real uth, + amrex::RandomEngine const& engine) + { + u_tang1 = (uth > 0._rt) ? PhysConst::c * amrex::RandomNormal(0._rt, uth, engine) : 0._rt; + u_tang2 = (uth > 0._rt) ? PhysConst::c * amrex::RandomNormal(0._rt, uth, engine) : 0._rt; + u_norm = (uth > 0._rt) ? std::copysign(1._prt, -u_norm) * PhysConst::c * generateGaussianFluxDist(0._rt, uth, engine) : 0._rt; + } + + + /* \brief Applies absorbing, reflecting or thermal boundary condition to the input particles, along all axis. * For reflecting boundaries, the position of the particle is changed appropriately and * the sign of the velocity is changed (depending on the reflect_all_velocities flag). * For absorbing, a flag is set whether the particle has been lost (it is up to the calling * code to take appropriate action to remove any lost particles). Absorbing boundaries can * be given a reflection coefficient for stochastic reflection of particles, this * coefficient is zero by default. + * For thermal boundaries, the particle is first reflected and the position of the particle + * is changed appropriately. * Note that periodic boundaries are handled in AMReX code. * * \param x, xmin, xmax: particle x position, location of x boundary @@ -103,21 +130,33 @@ namespace ApplyParticleBoundaries { bool change_sign_uz = false; #ifndef WARPX_DIM_1D_Z - apply_boundary(x, xmin, xmax, change_sign_ux, particle_lost, + bool rethermalize_x = false; // stores if particle crosses x boundary and needs to be thermalized + apply_boundary(x, xmin, xmax, change_sign_ux, rethermalize_x, particle_lost, boundaries.xmin_bc, boundaries.xmax_bc, boundaries.reflection_model_xlo(-ux), boundaries.reflection_model_xhi(ux), engine); + if (rethermalize_x) { + thermalize_boundary_particle(ux, uy, uz, boundaries.m_uth, engine); + } #endif #ifdef WARPX_DIM_3D - apply_boundary(y, ymin, ymax, change_sign_uy, particle_lost, + bool rethermalize_y = false; // stores if particle crosses y boundary and needs to be thermalized + apply_boundary(y, ymin, ymax, change_sign_uy, rethermalize_y, particle_lost, boundaries.ymin_bc, boundaries.ymax_bc, boundaries.reflection_model_ylo(-uy), boundaries.reflection_model_yhi(uy), engine); + if (rethermalize_y) { + thermalize_boundary_particle(uy, uz, ux, boundaries.m_uth, engine); + } #endif - apply_boundary(z, zmin, zmax, change_sign_uz, particle_lost, + bool rethermalize_z = false; // stores if particle crosses z boundary and needs to be thermalized + apply_boundary(z, zmin, zmax, change_sign_uz, rethermalize_z, particle_lost, boundaries.zmin_bc, boundaries.zmax_bc, boundaries.reflection_model_zlo(-uz), boundaries.reflection_model_zhi(uz), engine); + if (rethermalize_z) { + thermalize_boundary_particle(uz, ux, uy, boundaries.m_uth, engine); + } if (boundaries.reflect_all_velocities && (change_sign_ux | change_sign_uy | change_sign_uz)) { change_sign_ux = true; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 06154958dde..4adb5386a2a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -446,6 +446,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_boundary.query("reflect_all_velocities", flag); m_boundary_conditions.Set_reflect_all_velocities(flag); + // currently supports only isotropic thermal distribution + // same distribution is applied to all boundaries + const amrex::ParmParse pp_species_boundary("boundary." + species_name); + if (WarpX::isAnyParticleBoundaryThermal()) { + amrex::Real boundary_uth; + utils::parser::getWithParser(pp_species_boundary,"u_th",boundary_uth); + m_boundary_conditions.SetThermalVelocity(boundary_uth); + } } PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index b7af3121814..dd1ec66e051 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -160,7 +160,8 @@ enum struct ParticleBoundaryType { Absorbing = 0, //!< particles crossing domain boundary are removed Open = 1, //!< particles cross domain boundary leave with damped j Reflecting = 2, //!< particles are reflected - Periodic = 3 + Periodic = 3, //!< particles are introduced from the periodic boundary + Thermal = 4 }; /** MPI reductions diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 3092e1984d6..54c91e5eb15 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -134,6 +134,7 @@ const std::map ParticleBCType_algo_to_enum = {"open", ParticleBoundaryType::Open}, {"reflecting", ParticleBoundaryType::Reflecting}, {"periodic", ParticleBoundaryType::Periodic}, + {"thermal", ParticleBoundaryType::Thermal}, {"default", ParticleBoundaryType::Absorbing} }; diff --git a/Source/WarpX.H b/Source/WarpX.H index d9ca494e72b..1a56445f8dd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -197,12 +197,12 @@ public: static amrex::Vector field_boundary_hi; /** Integers that correspond to boundary condition applied to particles at the * lower domain boundaries - * (0 to 3 correspond to Absorbing, Open, Reflecting, Periodic) + * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ static amrex::Vector particle_boundary_lo; /** Integers that correspond to boundary condition applied to particles at the * upper domain boundaries - * (0 to 3 correspond to Absorbing, Open, Reflecting, Periodic) + * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ static amrex::Vector particle_boundary_hi; @@ -767,6 +767,8 @@ public: void CopyJPML (); bool isAnyBoundaryPML(); + /** True if any of the particle boundary condition type is Thermal */ + static bool isAnyParticleBoundaryThermal(); PML* GetPML (int lev); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5caeb9ffb2e..83f48394400 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3277,6 +3277,16 @@ WarpX::isAnyBoundaryPML() return false; } +bool +WarpX::isAnyParticleBoundaryThermal () +{ + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) {return true;} + if (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) {return true;} + } + return false; +} + std::string TagWithLevelSuffix (std::string name, int const level) { From a5d5432ddbd9a3e53389cc6c47c02c2459ca9192 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 1 Apr 2024 22:20:56 +0200 Subject: [PATCH 037/121] Clang tidy CI test: add performance-unnecessary-* checks to clang-tidy CI test (#4136) * clang-tidy CI test: add performance-unnecessary-* checks * fixed issues * address issues found with clang-tidy * address issues found with clang-tidy * change name for clarity * fixed bug * address issues found with clang-tidy * fix issue found with clang-tidy * fix bug * remove file apparently added due to merge conflict * fix bug * fix bug * addressed all issues found with clang-tidy * fix issues found with clang-tidy * revert space deletion * fix variable name * fix issues found with clang-tidy * fix some residual issues found with clang-tidy * Update Source/Diagnostics/BTDiagnostics.H Co-authored-by: David Grote --------- Co-authored-by: David Grote --- .clang-tidy | 2 -- Source/Diagnostics/BTD_Plotfile_Header_Impl.H | 6 ++-- .../Diagnostics/BTD_Plotfile_Header_Impl.cpp | 6 ++-- Source/Diagnostics/BTDiagnostics.H | 22 +++++++-------- Source/Diagnostics/BTDiagnostics.cpp | 24 ++++++++-------- .../Diagnostics/BoundaryScrapingDiagnostics.H | 2 +- .../BoundaryScrapingDiagnostics.cpp | 4 +-- .../ParticleReductionFunctor.H | 4 +-- .../ParticleReductionFunctor.cpp | 4 +-- .../FlushFormats/FlushFormatPlotfile.cpp | 4 +-- .../FlushFormats/FlushFormatSensei.H | 2 +- .../FlushFormats/FlushFormatSensei.cpp | 2 +- Source/Diagnostics/FullDiagnostics.H | 2 +- Source/Diagnostics/FullDiagnostics.cpp | 2 +- .../Diagnostics/ParticleDiag/ParticleDiag.H | 3 +- .../Diagnostics/ParticleDiag/ParticleDiag.cpp | 6 ++-- .../Diagnostics/ReducedDiags/BeamRelevant.H | 2 +- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 2 +- Source/Diagnostics/ReducedDiags/ChargeOnEB.H | 2 +- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 +- .../ReducedDiags/ColliderRelevant.H | 2 +- .../ReducedDiags/ColliderRelevant.cpp | 4 +-- Source/Diagnostics/ReducedDiags/FieldEnergy.H | 2 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldMaximum.H | 2 +- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldMomentum.H | 2 +- .../ReducedDiags/FieldMomentum.cpp | 2 +- Source/Diagnostics/ReducedDiags/FieldProbe.H | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldReduction.H | 2 +- .../ReducedDiags/FieldReduction.cpp | 2 +- .../ReducedDiags/LoadBalanceCosts.H | 2 +- .../ReducedDiags/LoadBalanceCosts.cpp | 2 +- .../ReducedDiags/LoadBalanceEfficiency.H | 2 +- .../ReducedDiags/LoadBalanceEfficiency.cpp | 2 +- .../Diagnostics/ReducedDiags/ParticleEnergy.H | 2 +- .../ReducedDiags/ParticleEnergy.cpp | 2 +- .../ReducedDiags/ParticleExtrema.H | 2 +- .../ReducedDiags/ParticleExtrema.cpp | 2 +- .../ReducedDiags/ParticleHistogram.H | 2 +- .../ReducedDiags/ParticleHistogram.cpp | 4 +-- .../ReducedDiags/ParticleHistogram2D.H | 2 +- .../ReducedDiags/ParticleHistogram2D.cpp | 2 +- .../ReducedDiags/ParticleMomentum.H | 2 +- .../ReducedDiags/ParticleMomentum.cpp | 2 +- .../Diagnostics/ReducedDiags/ParticleNumber.H | 2 +- .../ReducedDiags/ParticleNumber.cpp | 2 +- .../Diagnostics/ReducedDiags/ReducedDiags.H | 2 +- .../Diagnostics/ReducedDiags/ReducedDiags.cpp | 4 +-- Source/Diagnostics/ReducedDiags/RhoMaximum.H | 2 +- .../Diagnostics/ReducedDiags/RhoMaximum.cpp | 2 +- Source/Diagnostics/SliceDiagnostic.H | 8 +++--- Source/Diagnostics/SliceDiagnostic.cpp | 2 +- Source/Diagnostics/WarpXOpenPMD.H | 18 ++++++------ Source/Diagnostics/WarpXOpenPMD.cpp | 28 +++++++++---------- Source/FieldSolver/ElectrostaticSolver.H | 2 +- Source/Initialization/WarpXInitData.cpp | 6 ++-- Source/Laser/LaserProfiles.H | 4 +-- .../LaserProfileFromFile.cpp | 4 +-- Source/Parallelization/GuardCellManager.H | 4 +-- Source/Parallelization/GuardCellManager.cpp | 4 +-- .../BackgroundMCC/BackgroundMCCCollision.H | 2 +- .../BackgroundMCC/BackgroundMCCCollision.cpp | 2 +- .../BackgroundStopping/BackgroundStopping.H | 2 +- .../BackgroundStopping/BackgroundStopping.cpp | 2 +- .../BinaryCollision/BinaryCollisionUtils.H | 4 +-- .../BinaryCollision/BinaryCollisionUtils.cpp | 4 +-- .../Coulomb/PairWiseCoulombCollisionFunc.H | 2 +- .../Collision/BinaryCollision/DSMC/DSMC.H | 2 +- .../Collision/BinaryCollision/DSMC/DSMC.cpp | 4 +-- .../NuclearFusion/NuclearFusionFunc.H | 5 ++-- .../BinaryCollision/ParticleCreationFunc.H | 4 +-- .../BinaryCollision/ParticleCreationFunc.cpp | 7 ++--- Source/Particles/Collision/CollisionBase.H | 2 +- Source/Particles/Collision/CollisionBase.cpp | 2 +- .../Particles/Collision/ScatteringProcess.H | 2 +- .../Particles/Collision/ScatteringProcess.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 4 +-- Source/Particles/MultiParticleContainer.cpp | 4 +-- Source/Particles/ParticleBoundaryBuffer.H | 6 ++-- Source/Particles/ParticleBoundaryBuffer.cpp | 6 ++-- Source/Particles/PhysicalParticleContainer.H | 4 +-- .../Particles/PhysicalParticleContainer.cpp | 4 +-- .../Particles/Resampling/LevelingThinning.H | 2 +- .../Particles/Resampling/LevelingThinning.cpp | 2 +- Source/Particles/Resampling/Resampling.H | 2 +- Source/Particles/Resampling/Resampling.cpp | 2 +- .../Particles/Resampling/ResamplingTrigger.H | 2 +- .../Resampling/ResamplingTrigger.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 4 +-- Source/Python/callbacks.H | 8 +++--- Source/Python/callbacks.cpp | 10 +++---- Source/Utils/WarpXUtil.H | 4 +-- Source/Utils/WarpXUtil.cpp | 8 +++--- Source/WarpX.H | 4 +-- Source/ablastr/fields/PoissonSolver.H | 6 ++-- Source/ablastr/fields/VectorPoissonSolver.H | 6 ++-- Source/ablastr/utils/msg_logger/MsgLogger.H | 2 +- Source/ablastr/utils/msg_logger/MsgLogger.cpp | 2 +- Source/ablastr/warn_manager/WarnManager.H | 12 ++++---- Source/ablastr/warn_manager/WarnManager.cpp | 12 ++++---- 102 files changed, 209 insertions(+), 210 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 21f0343519c..5d2422af214 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -30,8 +30,6 @@ Checks: ' -modernize-use-trailing-return-type, mpi-*, performance-*, - -performance-unnecessary-copy-initialization, - -performance-unnecessary-value-param, portability-*, readability-*, -readability-convert-member-functions-to-static, diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 0a22774bb8d..9d3250fa52f 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -200,12 +200,12 @@ class BTDMultiFabHeaderImpl */ void SetBox (int ibox, amrex::Box ba_box) { m_ba.set(ibox, ba_box); } /** Set Fab name of the ith fab to be written in the multifab Header file.*/ - void SetFabName (int ifab, std::string fodPrefix, std::string FabName, + void SetFabName (int ifab, const std::string& fodPrefix, const std::string& FabName, int FabHead); /** Set minimum value of all the components for the ith fab. */ - void SetMinVal (int ifab, amrex::Vector minval); + void SetMinVal (int ifab, const amrex::Vector& minval); /** Set maximum value of all the components for the ith fab. */ - void SetMaxVal (int ifab, amrex::Vector maxval); + void SetMaxVal (int ifab, const amrex::Vector& maxval); private: /** Header file path */ std::string m_Header_path; diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index cd3eb962405..4ec1c974f8e 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -323,7 +323,7 @@ BTDMultiFabHeaderImpl::ResizeFabData () } void -BTDMultiFabHeaderImpl::SetFabName (int ifab, std::string fodPrefix, std::string FabName, +BTDMultiFabHeaderImpl::SetFabName (int ifab, const std::string& fodPrefix, const std::string& FabName, int FabHead) { m_FabOnDiskPrefix[ifab] = fodPrefix; @@ -333,13 +333,13 @@ BTDMultiFabHeaderImpl::SetFabName (int ifab, std::string fodPrefix, std::string } void -BTDMultiFabHeaderImpl::SetMinVal (int ifab, amrex::Vector minval) +BTDMultiFabHeaderImpl::SetMinVal (int ifab, const amrex::Vector& minval) { CopyVec(m_minval[ifab], minval); } void -BTDMultiFabHeaderImpl::SetMaxVal (int ifab, amrex::Vector maxval) +BTDMultiFabHeaderImpl::SetMaxVal (int ifab, const amrex::Vector& maxval) { CopyVec(m_maxval[ifab], maxval); } diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index f6c44c777ea..679dd6b8fa4 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -28,7 +28,7 @@ class BTDiagnostics final : public Diagnostics { public: - BTDiagnostics (int i, std::string name); + BTDiagnostics (int i, const std::string& name); private: /** Whether to plot raw (i.e., NOT cell-centered) fields */ @@ -375,26 +375,26 @@ private: /** Interleave lab-frame meta-data of the buffers to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveBufferAndSnapshotHeader ( std::string buffer_Header, - std::string snapshot_Header); + void InterleaveBufferAndSnapshotHeader (const std::string& buffer_Header, + const std::string& snapshot_Header); /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename); + void InterleaveFabArrayHeader (const std::string& Buffer_FabHeader_path, + const std::string& snapshot_FabHeader_path, + const std::string& newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data */ - void InterleaveSpeciesHeader(std::string buffer_species_Header_path, - std::string snapshot_species_Header_path, - std::string species_name, int new_data_index); + void InterleaveSpeciesHeader(const std::string& buffer_species_Header_path, + const std::string& snapshot_species_Header_path, + const std::string& species_name, int new_data_index); /** Interleave lab-frame metadata of the particle header file in the buffers to * be consistent with the merged plotfile lab-frame data */ - void InterleaveParticleDataHeader( std::string buffer_ParticleHdrFilename, - std::string snapshot_ParticleHdrFilename); + void InterleaveParticleDataHeader(const std::string& buffer_ParticleHdrFilename, + const std::string& snapshot_ParticleHdrFilename); /** Initialize particle functors for each species to compute the back-transformed lab-frame data. */ void InitializeParticleFunctors () override; diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index fd9c5c783c1..2422d1324af 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -52,8 +52,8 @@ namespace constexpr int permission_flag_rwxrxrx = 0755; } -BTDiagnostics::BTDiagnostics (int i, std::string name) - : Diagnostics(i, name) +BTDiagnostics::BTDiagnostics (int i, const std::string& name) + : Diagnostics{i, name} { ReadParameters(); } @@ -1290,8 +1290,8 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } void -BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_path, - std::string snapshot_Header_path) +BTDiagnostics::InterleaveBufferAndSnapshotHeader ( const std::string& buffer_Header_path, + const std::string& snapshot_Header_path) { BTDPlotfileHeaderImpl snapshot_HeaderImpl(snapshot_Header_path); snapshot_HeaderImpl.ReadHeaderData(); @@ -1335,9 +1335,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader(const std::string& Buffer_FabHeader_path, + const std::string& snapshot_FabHeader_path, + const std::string& newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); @@ -1365,9 +1365,9 @@ BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, } void -BTDiagnostics::InterleaveSpeciesHeader(std::string buffer_species_Header_path, - std::string snapshot_species_Header_path, - std::string species_name, const int new_data_index) +BTDiagnostics::InterleaveSpeciesHeader(const std::string& buffer_species_Header_path, + const std::string& snapshot_species_Header_path, + const std::string& species_name, const int new_data_index) { BTDSpeciesHeaderImpl BufferSpeciesHeader(buffer_species_Header_path, species_name); @@ -1389,8 +1389,8 @@ BTDiagnostics::InterleaveSpeciesHeader(std::string buffer_species_Header_path, } void -BTDiagnostics::InterleaveParticleDataHeader(std::string buffer_ParticleHdrFilename, - std::string snapshot_ParticleHdrFilename) +BTDiagnostics::InterleaveParticleDataHeader(const std::string& buffer_ParticleHdrFilename, + const std::string& snapshot_ParticleHdrFilename) { BTDParticleDataHeaderImpl BufferParticleHeader(buffer_ParticleHdrFilename); BufferParticleHeader.ReadHeader(); diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 60d184c30e2..3e5fc1f19eb 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -23,7 +23,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - BoundaryScrapingDiagnostics (int i, std::string name); + BoundaryScrapingDiagnostics (int i, const std::string& name); private: /** Read relevant parameters for BoundaryScraping */ diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 11ffce02f09..da1e5fdcc00 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -21,8 +21,8 @@ using namespace amrex::literals; -BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, std::string name) - : Diagnostics(i, name) +BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name) + : Diagnostics{i, name} { ReadParameters(); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 56c4372fad8..7de9844a99e 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -30,9 +30,9 @@ public: * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ ParticleReductionFunctor(const amrex::MultiFab * mf_src, int lev, - amrex::IntVect crse_ratio, std::string fn_str, + amrex::IntVect crse_ratio, const std::string& fn_str, int ispec, bool do_average, - bool do_filter, std::string filter_str, + bool do_filter, const std::string& filter_str, int ncomp=1); /** \brief Compute the average of the function m_map_fn over each grid cell. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index feb25c11183..15473890233 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -18,9 +18,9 @@ using namespace amrex::literals; ParticleReductionFunctor::ParticleReductionFunctor (const amrex::MultiFab* mf_src, const int lev, - const amrex::IntVect crse_ratio, const std::string fn_str, + const amrex::IntVect crse_ratio, const std::string& fn_str, const int ispec, const bool do_average, - const bool do_filter, const std::string filter_str, const int ncomp) + const bool do_filter, const std::string& filter_str, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), m_ispec(ispec), m_do_average(do_average), m_do_filter(do_filter) { // mf_src will not be used, let's make sure it's null. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 48361d90fc0..555ffb123e7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -479,7 +479,7 @@ WriteZeroRawMF( const MultiFab& F, const DistributionMapping& dm, * coarse and fine patch to have the same shape. */ void -WriteCoarseVector( const std::string field_name, +WriteCoarseVector( const std::string& field_name, const MultiFab* Fx_cp, const MultiFab* Fy_cp, const MultiFab* Fz_cp, @@ -519,7 +519,7 @@ WriteCoarseVector( const std::string field_name, * coarse and fine patch to have the same shape. */ void -WriteCoarseScalar( const std::string field_name, +WriteCoarseScalar( const std::string& field_name, const MultiFab* F_cp, const MultiFab* F_fp, const DistributionMapping& dm, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index d2ec9a5a4e0..45ea40077e4 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -45,7 +45,7 @@ public: * \param[in] amr_mesh an AmrMesh instance * \param[in] diag_name ParmParse scope string. */ - FlushFormatSensei (amrex::AmrMesh *amr_mesh, std::string diag_name); + FlushFormatSensei (amrex::AmrMesh *amr_mesh, const std::string& diag_name); /** Do in-situ visualization for field and particle data */ void WriteToFile ( diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index 348e1da4a00..468ed81ce18 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -8,7 +8,7 @@ #endif FlushFormatSensei::FlushFormatSensei (amrex::AmrMesh *amr_mesh, - std::string diag_name) : + const std::string& diag_name) : m_amr_mesh(amr_mesh) { #ifndef AMREX_USE_SENSEI_INSITU diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 42b60c213f2..1b999a9b361 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -9,7 +9,7 @@ class FullDiagnostics final : public Diagnostics { public: - FullDiagnostics (int i, std::string name); + FullDiagnostics (int i, const std::string& name); private: /** Read user-requested parameters for full diagnostics */ void ReadParameters (); diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 27fe7e48c66..bf4cd3f95fc 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -41,7 +41,7 @@ using namespace amrex::literals; -FullDiagnostics::FullDiagnostics (int i, std::string name): +FullDiagnostics::FullDiagnostics (int i, const std::string& name): Diagnostics{i, name}, m_solver_deposits_current{ (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None) || diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.H b/Source/Diagnostics/ParticleDiag/ParticleDiag.H index 7d8b5819f5f..e9f54d2e5e1 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.H +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.H @@ -17,7 +17,8 @@ class ParticleDiag { public: - ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); + ParticleDiag(const std::string& diag_name, const std::string& name, + WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); [[nodiscard]] WarpXParticleContainer* getParticleContainer() const { return m_pc; } [[nodiscard]] PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } [[nodiscard]] std::string getSpeciesName() const { return m_name; } diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 18fbb4f593b..9954fa24285 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -13,8 +13,10 @@ using namespace amrex; -ParticleDiag::ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc) - : m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) +ParticleDiag::ParticleDiag( + const std::string& diag_name, const std::string& name, + WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc): + m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) { //variable to set m_plot_flags size const int plot_flag_size = pc->NumRealComps(); diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.H b/Source/Diagnostics/ReducedDiags/BeamRelevant.H index 018ddef5463..ed410c72fc9 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.H +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.H @@ -23,7 +23,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - BeamRelevant(std::string rd_name); + BeamRelevant(const std::string& rd_name); /// name of beam species std::string m_beam_name; diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index b78a7713abc..ae7d5230b4c 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -32,7 +32,7 @@ using namespace amrex; // constructor -BeamRelevant::BeamRelevant (std::string rd_name) +BeamRelevant::BeamRelevant (const std::string& rd_name) : ReducedDiags{rd_name} { // read beam name diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H index ed69ce01767..3cea5b0b30e 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H @@ -33,7 +33,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ChargeOnEB (std::string rd_name); + ChargeOnEB (const std::string& rd_name); /** * This function computes the charge at the surface of the EB: diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index c6f75e4a712..1398072a84d 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -28,7 +28,7 @@ using namespace amrex; // constructor -ChargeOnEB::ChargeOnEB (std::string rd_name) +ChargeOnEB::ChargeOnEB (const std::string& rd_name) : ReducedDiags{rd_name} { // Only 3D is working for now diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H index fe00dd2e7e3..3b93eb1ed25 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H @@ -26,7 +26,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ColliderRelevant(std::string rd_name); + ColliderRelevant(const std::string& rd_name); /// name of the two colliding species std::vector m_beam_name; diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 4cd1a66f3f2..3229f10c89e 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -59,8 +59,8 @@ using namespace amrex; -ColliderRelevant::ColliderRelevant (std::string rd_name) -: ReducedDiags{std::move(rd_name)} +ColliderRelevant::ColliderRelevant (const std::string& rd_name) +: ReducedDiags{rd_name} { // read colliding species names - must be 2 const amrex::ParmParse pp_rd_name(m_rd_name); diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.H b/Source/Diagnostics/ReducedDiags/FieldEnergy.H index 4f5d1656841..40de174526e 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.H +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.H @@ -26,7 +26,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldEnergy(std::string rd_name); + FieldEnergy(const std::string& rd_name); /** * This function computes the field energy (EF): diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index a88690f863f..1271e285c6c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -30,7 +30,7 @@ using namespace amrex; // constructor -FieldEnergy::FieldEnergy (std::string rd_name) +FieldEnergy::FieldEnergy (const std::string& rd_name) : ReducedDiags{rd_name} { diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.H b/Source/Diagnostics/ReducedDiags/FieldMaximum.H index 95ae6ff16da..1d7cf4d236c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldMaximum(std::string rd_name); + FieldMaximum(const std::string& rd_name); /** * This function computes the maximum value of Ex, Ey, Ez, |E|, Bx, By, Bz and |B| diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 6465ee2be1b..8c66404056b 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -40,7 +40,7 @@ using namespace amrex; // constructor -FieldMaximum::FieldMaximum (std::string rd_name) +FieldMaximum::FieldMaximum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.H b/Source/Diagnostics/ReducedDiags/FieldMomentum.H index 0f04b6413d1..99eabf3772f 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.H @@ -24,7 +24,7 @@ public: * * \param[in] rd_name reduced diags names */ - FieldMomentum(std::string rd_name); + FieldMomentum(const std::string& rd_name); /** * \brief This function computes the electromagnetic momentum, diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index fe2040e50d2..3b3b7463593 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -38,7 +38,7 @@ using namespace amrex; -FieldMomentum::FieldMomentum (std::string rd_name) +FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.H b/Source/Diagnostics/ReducedDiags/FieldProbe.H index 9f318c4f8a0..a7c04649999 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.H @@ -41,7 +41,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldProbe (std::string rd_name); + FieldProbe (const std::string& rd_name); /** * This function assins test/data particles to constructed environemnt diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 24ad0e64ea8..f0780622d54 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -47,7 +47,7 @@ using namespace amrex; // constructor -FieldProbe::FieldProbe (std::string rd_name) +FieldProbe::FieldProbe (const std::string& rd_name) : ReducedDiags{rd_name}, m_probe(&WarpX::GetInstance()) { diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index e99c87ad14f..940222867b1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -50,7 +50,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldReduction(std::string rd_name); + FieldReduction(const std::string& rd_name); /** * This function is called at every time step, and if necessary calls the templated function diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp index 85e4d4028e6..683ae1921d6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp @@ -22,7 +22,7 @@ #include // constructor -FieldReduction::FieldReduction (std::string rd_name) +FieldReduction::FieldReduction (const std::string& rd_name) : ReducedDiags{rd_name} { using namespace amrex::literals; diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H index 0ecb74d4bf5..cfba804d0f0 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H @@ -54,7 +54,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - LoadBalanceCosts(std::string rd_name); + LoadBalanceCosts(const std::string& rd_name); /** * This function updates the costs, given the current distribution mapping, diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index b4e07b51982..8af2c66711c 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -65,7 +65,7 @@ namespace } // constructor -LoadBalanceCosts::LoadBalanceCosts (std::string rd_name) +LoadBalanceCosts::LoadBalanceCosts (const std::string& rd_name) : ReducedDiags{rd_name} { } diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H index a43868ccc5f..c58f78deedb 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - LoadBalanceEfficiency(std::string rd_name); + LoadBalanceEfficiency(const std::string& rd_name); /** * This function gets the current load balance efficiency diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp index 6dd066d5324..b6f3a58e670 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp @@ -20,7 +20,7 @@ using namespace amrex; // constructor -LoadBalanceEfficiency::LoadBalanceEfficiency (std::string rd_name) +LoadBalanceEfficiency::LoadBalanceEfficiency (const std::string& rd_name) : ReducedDiags{rd_name} { // read number of levels diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H index b0df66709f3..cb3fb8c4f08 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H @@ -25,7 +25,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleEnergy(std::string rd_name); + ParticleEnergy(const std::string& rd_name); /** * This function computes the particle relativistic kinetic energy (EP). diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index 157b4bfb368..9446af5e20c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -34,7 +34,7 @@ using namespace amrex; // constructor -ParticleEnergy::ParticleEnergy (std::string rd_name) +ParticleEnergy::ParticleEnergy (const std::string& rd_name) : ReducedDiags{rd_name} { // get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H index ed5cce1d7e0..d3505b5ccf7 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H @@ -25,7 +25,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleExtrema(std::string rd_name); + ParticleExtrema(const std::string& rd_name); /// name of species std::string m_species_name; diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index ba8e7e373de..0414d9ce869 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -53,7 +53,7 @@ using namespace amrex; // constructor -ParticleExtrema::ParticleExtrema (std::string rd_name) +ParticleExtrema::ParticleExtrema (const std::string& rd_name) : ReducedDiags{rd_name} { // read species name diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H index 758f4a399a9..39ccd8f1ace 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H @@ -28,7 +28,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleHistogram(std::string rd_name); + ParticleHistogram(const std::string& rd_name); /// normalization type int m_norm; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index 512e6f1394a..4aeb2af526f 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -50,8 +50,8 @@ struct NormalizationType { }; // constructor -ParticleHistogram::ParticleHistogram (std::string rd_name): - ReducedDiags{rd_name} +ParticleHistogram::ParticleHistogram (const std::string& rd_name) +: ReducedDiags{rd_name} { const ParmParse pp_rd_name(rd_name); diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H index 6ba85502819..0c648528920 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H @@ -30,7 +30,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleHistogram2D(std::string rd_name); + ParticleHistogram2D(const std::string& rd_name); /// File type std::string m_openpmd_backend {"default"}; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index 0d696f8b7bc..8dd19186b25 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -54,7 +54,7 @@ using namespace amrex; // constructor -ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) +ParticleHistogram2D::ParticleHistogram2D (const std::string& rd_name) : ReducedDiags{rd_name} { ParmParse pp_rd_name(rd_name); diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H index feb8957e516..ea079f92e62 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H @@ -25,7 +25,7 @@ public: * * \param[in] rd_name reduced diags names */ - ParticleMomentum(std::string rd_name); + ParticleMomentum(const std::string& rd_name); /** * \brief This function computes the particle relativistic momentum, diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5f278d48f14..47d1ec2cdd4 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -32,7 +32,7 @@ using namespace amrex; -ParticleMomentum::ParticleMomentum (std::string rd_name) +ParticleMomentum::ParticleMomentum (const std::string& rd_name) : ReducedDiags{rd_name} { // Get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.H b/Source/Diagnostics/ReducedDiags/ParticleNumber.H index 3143a387a74..894048806ca 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.H +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleNumber(std::string rd_name); + ParticleNumber(const std::string& rd_name); /** * This function computes the total number of macroparticles and physical particles of each diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp index 7fe3f75d4ce..bfdf1daffe3 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp @@ -27,7 +27,7 @@ using namespace amrex::literals; // constructor -ParticleNumber::ParticleNumber (std::string rd_name) +ParticleNumber::ParticleNumber (const std::string& rd_name) : ReducedDiags{rd_name} { // get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 09aa85586be..2c942e1df6d 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -52,7 +52,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ReducedDiags (std::string rd_name); + ReducedDiags (const std::string& rd_name); /** * Virtual destructor for polymorphism diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 483f4c68ece..e3835dde416 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -23,8 +23,8 @@ using namespace amrex; // constructor -ReducedDiags::ReducedDiags (std::string rd_name) - : m_rd_name(std::move(rd_name)) +ReducedDiags::ReducedDiags (const std::string& rd_name): +m_rd_name{rd_name} { BackwardCompatibility(); diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.H b/Source/Diagnostics/ReducedDiags/RhoMaximum.H index 6f306a5e7ad..ceff0f666e0 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.H +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.H @@ -28,7 +28,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - RhoMaximum(std::string rd_name); + RhoMaximum(const std::string& rd_name); /** * This function computes the maximum and minimum values of rho (summed over all species) and diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 4613a70b3b0..9902415d9a9 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -29,7 +29,7 @@ using namespace amrex::literals; // constructor -RhoMaximum::RhoMaximum (std::string rd_name) +RhoMaximum::RhoMaximum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/SliceDiagnostic.H b/Source/Diagnostics/SliceDiagnostic.H index 000b01d470f..570f86d5384 100644 --- a/Source/Diagnostics/SliceDiagnostic.H +++ b/Source/Diagnostics/SliceDiagnostic.H @@ -25,10 +25,10 @@ void CheckSliceInput( amrex::RealBox real_box, amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); void InterpolateSliceValues( amrex::MultiFab& smf, - amrex::IntVect interp_lo, amrex::RealBox slice_realbox, - amrex::Vector geom, int ncomp, int nghost, - amrex::IntVect slice_lo, amrex::IntVect slice_hi, - amrex::IntVect SliceType, amrex::RealBox real_box); + amrex::IntVect interp_lo, amrex::RealBox slice_realbox, + const amrex::Vector& geom, int ncomp, int nghost, + amrex::IntVect slice_lo, amrex::IntVect slice_hi, + amrex::IntVect SliceType, amrex::RealBox real_box); void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, amrex::IntVect slice_lo, amrex::Vector geom, diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 71a585aea35..f373e78c9b2 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -419,7 +419,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, */ void InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, - Vector geom, int ncomp, int nghost, + const Vector& geom, int ncomp, int nghost, IntVect slice_lo, IntVect /*slice_hi*/, IntVect SliceType, const RealBox real_box) { diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 6c904790e15..12aa46de058 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -91,12 +91,12 @@ public: * @param authors a string specifying the authors of the simulation (can be empty) */ WarpXOpenPMDPlot (openPMD::IterationEncoding ie, - std::string filetype, - std::string operator_type, - std::map< std::string, std::string > operator_parameters, - std::string engine_type, - std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections, + const std::string& filetype, + const std::string& operator_type, + const std::map< std::string, std::string >& operator_parameters, + const std::string& engine_type, + const std::map< std::string, std::string >& engine_parameters, + const std::vector& fieldPMLdirections, const std::string& authors); ~WarpXOpenPMDPlot (); @@ -188,9 +188,9 @@ private: void SetupMeshComp ( openPMD::Mesh& mesh, - amrex::Geometry& full_geom, - std::string comp_name, - std::string field_name, + amrex::Geometry const& full_geom, + std::string const& comp_name, + std::string const& field_name, amrex::MultiFab const& mf, bool var_in_theta_mode ) const; diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index b55dbe79175..d90f3b45c96 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -343,7 +343,7 @@ namespace detail * set the metadata that indicates the physical unit. */ inline void - setOpenPMDUnit ( openPMD::Mesh mesh, const std::string field_name ) + setOpenPMDUnit ( openPMD::Mesh mesh, const std::string& field_name ) { if (field_name[0] == 'E'){ // Electric field mesh.setUnitDimension({ @@ -377,19 +377,19 @@ namespace detail #ifdef WARPX_USE_OPENPMD WarpXOpenPMDPlot::WarpXOpenPMDPlot ( openPMD::IterationEncoding ie, - std::string openPMDFileType, - std::string operator_type, - std::map< std::string, std::string > operator_parameters, - std::string engine_type, - std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections, + const std::string& openPMDFileType, + const std::string& operator_type, + const std::map< std::string, std::string >& operator_parameters, + const std::string& engine_type, + const std::map< std::string, std::string >& engine_parameters, + const std::vector& fieldPMLdirections, const std::string& authors) : m_Series(nullptr), m_MPIRank{amrex::ParallelDescriptor::MyProc()}, m_MPISize{amrex::ParallelDescriptor::NProcs()}, m_Encoding(ie), - m_OpenPMDFileType(std::move(openPMDFileType)), - m_fieldPMLdirections(std::move(fieldPMLdirections)), + m_OpenPMDFileType{openPMDFileType}, + m_fieldPMLdirections{fieldPMLdirections}, m_authors{authors} { m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters, @@ -816,7 +816,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, // the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names // so using the min to be safe. // - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + auto const getComponentRecord = [&currSpecies](std::string const& comp_name) { // handle scalar and non-scalar records by name const auto [record_name, component_name] = detail::name2openPMD(comp_name); return currSpecies[record_name][component_name]; @@ -893,7 +893,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, auto const numParticleOnTile64 = static_cast(numParticleOnTile); auto const& soa = pti.GetStructOfArrays(); - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + auto const getComponentRecord = [&currSpecies](std::string const& comp_name) { // handle scalar and non-scalar records by name const auto [record_name, component_name] = detail::name2openPMD(comp_name); return currSpecies[record_name][component_name]; @@ -1194,9 +1194,9 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, */ void WarpXOpenPMDPlot::SetupMeshComp (openPMD::Mesh& mesh, - amrex::Geometry& full_geom, - std::string comp_name, - std::string field_name, + amrex::Geometry const& full_geom, + std::string const& comp_name, + std::string const& field_name, amrex::MultiFab const& mf, bool var_in_theta_mode) const { diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 7fc6c9bf6da..0a2abe2d6b2 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -56,7 +56,7 @@ namespace ElectrostaticSolver { * * \param [in] potential The string value of the potential */ - void setPotentialEB (std::string potential) { + void setPotentialEB(const std::string& potential) { potential_eb_str = potential; buildParsersEB(); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e3ec624be8e..5309b878f08 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1337,8 +1337,8 @@ WarpX::LoadExternalFieldsFromFile (int const lev) #if defined(WARPX_USE_OPENPMD) && !defined(WARPX_DIM_1D_Z) && !defined(WARPX_DIM_XZ) void WarpX::ReadExternalFieldFromFile ( - std::string read_fields_from_path, amrex::MultiFab* mf, - std::string F_name, std::string F_component) + const std::string& read_fields_from_path, amrex::MultiFab* mf, + const std::string& F_name, const std::string& F_component) { // Get WarpX domain info auto& warpx = WarpX::GetInstance(); @@ -1514,7 +1514,7 @@ WarpX::ReadExternalFieldFromFile ( } // End function WarpX::ReadExternalFieldFromFile #else // WARPX_USE_OPENPMD && !WARPX_DIM_1D_Z && !defined(WARPX_DIM_XZ) void -WarpX::ReadExternalFieldFromFile (std::string , amrex::MultiFab* ,std::string, std::string) +WarpX::ReadExternalFieldFromFile (const std::string& , amrex::MultiFab* , const std::string& , const std::string& ) { #if defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE("Reading fields from openPMD files is not supported in 1D"); diff --git a/Source/Laser/LaserProfiles.H b/Source/Laser/LaserProfiles.H index 3aea88ac956..c2a3f337fbf 100644 --- a/Source/Laser/LaserProfiles.H +++ b/Source/Laser/LaserProfiles.H @@ -286,7 +286,7 @@ private: /** \brief parse a field file in the HDF5 'lasy' format * \param lasy_file_name: name of the lasy file to parse */ - void parse_lasy_file(std::string lasy_file_name); + void parse_lasy_file(const std::string& lasy_file_name); /** \brief parse a field file in the binary 'binary' format (whose details are given below). * @@ -301,7 +301,7 @@ private: * The spatiotemporal grid must be rectangular and uniform. * \param binary_file_name: name of the binary file to parse */ - void parse_binary_file(std::string binary_file_name); + void parse_binary_file(const std::string& binary_file_name); /** \brief Finds left and right time indices corresponding to time t * diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index 0fdb45c64f8..934a537be5a 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -162,7 +162,7 @@ WarpXLaserProfiles::FromFileLaserProfile::fill_amplitude ( } void -WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_name) +WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(const std::string& lasy_file_name) { #ifdef WARPX_USE_OPENPMD if(ParallelDescriptor::IOProcessor()){ @@ -237,7 +237,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ } void -WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_file_name) +WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (const std::string& binary_file_name) { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 38cef54921b..5c530441fc9 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -63,8 +63,8 @@ public: int nci_corr_stencil, int electromagnetic_solver_id, int max_level, - amrex::Vector v_galilean, - amrex::Vector v_comoving, + const amrex::Vector& v_galilean, + const amrex::Vector& v_comoving, bool safe_guard_cells, int do_multi_J, bool fft_do_time_averaging, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index ae5ab839d10..dc7800f490a 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -44,8 +44,8 @@ guardCellManager::Init ( const int nci_corr_stencil, const int electromagnetic_solver_id, const int max_level, - const amrex::Vector v_galilean, - const amrex::Vector v_comoving, + const amrex::Vector& v_galilean, + const amrex::Vector& v_comoving, const bool safe_guard_cells, const int do_multi_J, const bool fft_do_time_averaging, diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index 1de6b999e0b..3fd8b4d6fc2 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -23,7 +23,7 @@ class BackgroundMCCCollision final : public CollisionBase { public: - BackgroundMCCCollision (std::string collision_name); + BackgroundMCCCollision (std::string const& collision_name); ~BackgroundMCCCollision () override = default; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 4cb16f6fa50..1eb89ff4c0f 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -21,7 +21,7 @@ #include -BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name) +BackgroundMCCCollision::BackgroundMCCCollision (std::string const& collision_name) : CollisionBase(collision_name) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_species_names.size() == 1, diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H index 55fa4b9e1e3..8ba9b6f1b31 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H @@ -24,7 +24,7 @@ class BackgroundStopping final : public CollisionBase { public: - BackgroundStopping (std::string collision_name); + BackgroundStopping (const std::string& collision_name); ~BackgroundStopping () override = default; diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index 8122d7225b4..517b1138295 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -16,7 +16,7 @@ #include -BackgroundStopping::BackgroundStopping (std::string const collision_name) +BackgroundStopping::BackgroundStopping (std::string const& collision_name) : CollisionBase(collision_name) { using namespace amrex::literals; diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index b8a390ddb93..f6aa8e6e303 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -31,10 +31,10 @@ enum struct NuclearFusionType { namespace BinaryCollisionUtils{ - NuclearFusionType get_nuclear_fusion_type (std::string collision_name, + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * mypc); - CollisionType get_collision_type (std::string collision_name, + CollisionType get_collision_type (const std::string& collision_name, MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 430bac5e548..812be089462 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -17,7 +17,7 @@ namespace BinaryCollisionUtils{ - NuclearFusionType get_nuclear_fusion_type (const std::string collision_name, + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * const mypc) { const amrex::ParmParse pp_collision_name(collision_name); @@ -102,7 +102,7 @@ namespace BinaryCollisionUtils{ return NuclearFusionType::Undefined; } - CollisionType get_collision_type (const std::string collision_name, + CollisionType get_collision_type (const std::string& collision_name, MultiParticleContainer const * const mypc) { const amrex::ParmParse pp_collision_name(collision_name); diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index cfdc36d3c50..d1a92ee9716 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -46,7 +46,7 @@ public: * @param[in] mypc the particle container (unused) * @param[in] isSameSpecies true if this is an intra-species colission */ - PairWiseCoulombCollisionFunc (const std::string collision_name, + PairWiseCoulombCollisionFunc (const std::string& collision_name, [[maybe_unused]] MultiParticleContainer const * const mypc, const bool isSameSpecies): m_isSameSpecies{isSameSpecies} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H index ab01eba2c81..aa656a3d92e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -50,7 +50,7 @@ public: * * @param[in] collision_name the name of the collision */ - DSMC (std::string collision_name); + DSMC (const std::string& collision_name); /** Perform the collisions * diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp index 92ae3ddf257..8f309c3610a 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp @@ -11,8 +11,8 @@ #include "SplitAndScatterFunc.H" -DSMC::DSMC (const std::string collision_name) - : CollisionBase(collision_name) +DSMC::DSMC (const std::string& collision_name) + : CollisionBase{collision_name} { using namespace amrex::literals; const amrex::ParmParse pp_collision_name(collision_name); diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b2a2112ca68..8f19364baec 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -56,11 +56,12 @@ public: * @param[in] mypc pointer to the MultiParticleContainer * @param[in] isSameSpecies whether the two colliding species are the same */ - NuclearFusionFunc (const std::string collision_name, MultiParticleContainer const * const mypc, + NuclearFusionFunc (const std::string& collision_name, MultiParticleContainer const * const mypc, const bool isSameSpecies): m_fusion_multiplier{amrex::ParticleReal{1.0}}, // default fusion multiplier m_probability_threshold{amrex::ParticleReal{0.02}}, // default fusion probability threshold m_probability_target_value{amrex::ParticleReal{0.002}}, // default fusion probability target_value + m_fusion_type{BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc)}, m_isSameSpecies{isSameSpecies} { @@ -68,8 +69,6 @@ public: WARPX_ABORT_WITH_MESSAGE("Nuclear fusion module does not currently work with single precision"); #endif - m_fusion_type = BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc); - const amrex::ParmParse pp_collision_name(collision_name); utils::parser::queryWithParser( pp_collision_name, "fusion_multiplier", m_fusion_multiplier); diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index d53ce4348fb..4a77006dd92 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -52,7 +52,7 @@ public: * @param[in] collision_name the name of the collision * @param[in] mypc pointer to the MultiParticleContainer */ - ParticleCreationFunc (std::string collision_name, MultiParticleContainer const * mypc); + ParticleCreationFunc (const std::string& collision_name, MultiParticleContainer const * mypc); /** * \brief operator() of the ParticleCreationFunc class. It creates new particles from binary @@ -318,7 +318,7 @@ class NoParticleCreationFunc public: NoParticleCreationFunc () = default; - NoParticleCreationFunc (const std::string /*collision_name*/, + NoParticleCreationFunc (const std::string& /*collision_name*/, MultiParticleContainer const * const /*mypc*/) {} AMREX_INLINE diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 33629cd530f..3690c5d7295 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -17,13 +17,12 @@ #include -ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, - MultiParticleContainer const * const mypc) +ParticleCreationFunc::ParticleCreationFunc (const std::string& collision_name, + MultiParticleContainer const * const mypc): + m_collision_type{BinaryCollisionUtils::get_collision_type(collision_name, mypc)} { const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) { // Proton-Boron fusion only produces alpha particles diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index db79eaf1a01..21a5de6a38c 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -18,7 +18,7 @@ class CollisionBase { public: - CollisionBase (std::string collision_name); + CollisionBase (const std::string& collision_name); virtual void doCollisions (amrex::Real /*cur_time*/, amrex::Real /*dt*/, MultiParticleContainer* /*mypc*/ ){} diff --git a/Source/Particles/Collision/CollisionBase.cpp b/Source/Particles/Collision/CollisionBase.cpp index 8f799c83013..c8b4f76cbd2 100644 --- a/Source/Particles/Collision/CollisionBase.cpp +++ b/Source/Particles/Collision/CollisionBase.cpp @@ -10,7 +10,7 @@ #include -CollisionBase::CollisionBase (std::string collision_name) +CollisionBase::CollisionBase (const std::string& collision_name) { // read collision species diff --git a/Source/Particles/Collision/ScatteringProcess.H b/Source/Particles/Collision/ScatteringProcess.H index d05ea32e2fe..59ef7a02afb 100644 --- a/Source/Particles/Collision/ScatteringProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -57,7 +57,7 @@ public: */ static void readCrossSectionFile ( - std::string cross_section_file, + const std::string& cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ); diff --git a/Source/Particles/Collision/ScatteringProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp index c15d11eea07..ea1b4b40f54 100644 --- a/Source/Particles/Collision/ScatteringProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -94,7 +94,7 @@ ScatteringProcess::parseProcessType(const std::string& scattering_process) void ScatteringProcess::readCrossSectionFile ( - const std::string cross_section_file, + const std::string& cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) { diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 9946a680fcd..0421febc475 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -249,7 +249,7 @@ public: * \param[in] species_name The species for which back-transformed particles is set. * \param[in] do_back_transformed_particles The parameter to set if back-transformed particles are set to true/false */ - void SetDoBackTransformedParticles (std::string species_name, bool do_back_transformed_particles); + void SetDoBackTransformedParticles (const std::string& species_name, bool do_back_transformed_particles); [[nodiscard]] int nSpeciesDepositOnMainGrid () const { @@ -330,7 +330,7 @@ public: const amrex::MultiFab& Bz); #endif - [[nodiscard]] int getSpeciesID (std::string product_str) const; + [[nodiscard]] int getSpeciesID (const std::string& product_str) const; amrex::Vector>::iterator begin() {return allcontainers.begin();} amrex::Vector>::iterator end() {return allcontainers.end();} diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a31f426a0e4..b70844a8d15 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -803,7 +803,7 @@ MultiParticleContainer::mapSpeciesProduct () /* \brief Given a species name, return its ID. */ int -MultiParticleContainer::getSpeciesID (std::string product_str) const +MultiParticleContainer::getSpeciesID (const std::string& product_str) const { auto species_and_lasers_names = GetSpeciesAndLasersNames(); int i_product = 0; @@ -832,7 +832,7 @@ MultiParticleContainer::SetDoBackTransformedParticles (const bool do_back_transf } void -MultiParticleContainer::SetDoBackTransformedParticles (std::string species_name, const bool do_back_transformed_particles) { +MultiParticleContainer::SetDoBackTransformedParticles (const std::string& species_name, const bool do_back_transformed_particles) { auto species_names_list = GetSpeciesNames(); bool found = false; // Loop over species diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 1e9748b2ff5..20bc63f7a19 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -48,11 +48,11 @@ public: void printNumParticles () const; - int getNumParticlesInContainer(std::string species_name, int boundary, bool local); + int getNumParticlesInContainer(const std::string& species_name, int boundary, bool local); - PinnedMemoryParticleContainer& getParticleBuffer(std::string species_name, int boundary); + PinnedMemoryParticleContainer& getParticleBuffer(const std::string& species_name, int boundary); - PinnedMemoryParticleContainer* getParticleBufferPointer(std::string species_name, int boundary); + PinnedMemoryParticleContainer* getParticleBufferPointer(const std::string& species_name, int boundary); static constexpr int numBoundaries () { return AMREX_SPACEDIM*2 diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index d1f9f814bc5..25b77ddc496 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -531,7 +531,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } int ParticleBoundaryBuffer::getNumParticlesInContainer( - const std::string species_name, int boundary, bool local) { + const std::string& species_name, int boundary, bool local) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); @@ -545,7 +545,7 @@ int ParticleBoundaryBuffer::getNumParticlesInContainer( } PinnedMemoryParticleContainer & -ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int boundary) { +ParticleBoundaryBuffer::getParticleBuffer(const std::string& species_name, int boundary) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); @@ -560,7 +560,7 @@ ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int bo } PinnedMemoryParticleContainer * -ParticleBoundaryBuffer::getParticleBufferPointer(const std::string species_name, int boundary) { +ParticleBoundaryBuffer::getParticleBufferPointer(const std::string& species_name, int boundary) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 286d0675da6..5d9b41b8b75 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -352,14 +352,14 @@ public: * @param[in] ptr the pointer */ void set_breit_wheeler_engine_ptr - (std::shared_ptr ptr) override; + (const std::shared_ptr& ptr) override; /** * Acquires a shared smart pointer to a QuantumSynchrotronEngine * @param[in] ptr the pointer */ void set_quantum_sync_engine_ptr - (std::shared_ptr ptr) override; + (const std::shared_ptr& ptr) override; //__________ BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4adb5386a2a..9923d93a907 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -3392,14 +3392,14 @@ bool PhysicalParticleContainer::has_breit_wheeler () const void PhysicalParticleContainer:: -set_breit_wheeler_engine_ptr (std::shared_ptr ptr) +set_breit_wheeler_engine_ptr (const std::shared_ptr& ptr) { m_shr_p_bw_engine = ptr; } void PhysicalParticleContainer:: -set_quantum_sync_engine_ptr (std::shared_ptr ptr) +set_quantum_sync_engine_ptr (const std::shared_ptr& ptr) { m_shr_p_qs_engine = ptr; } diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index fa05525e270..9e3e63fc8be 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -36,7 +36,7 @@ public: * * @param[in] species_name the name of the resampled species */ - LevelingThinning (std::string species_name); + LevelingThinning (const std::string& species_name); /** * \brief A method that performs leveling thinning for the considered species. diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 5dc6a458f97..71ebe735c78 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -29,7 +29,7 @@ #include -LevelingThinning::LevelingThinning (const std::string species_name) +LevelingThinning::LevelingThinning (const std::string& species_name) { using namespace amrex::literals; diff --git a/Source/Particles/Resampling/Resampling.H b/Source/Particles/Resampling/Resampling.H index 35439e905df..7c5c60d858d 100644 --- a/Source/Particles/Resampling/Resampling.H +++ b/Source/Particles/Resampling/Resampling.H @@ -62,7 +62,7 @@ public: * * @param[in] species_name the name of the resampled species */ - Resampling (std::string species_name); + Resampling (const std::string& species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/Resampling.cpp b/Source/Particles/Resampling/Resampling.cpp index 63ac448880b..11f8706dc33 100644 --- a/Source/Particles/Resampling/Resampling.cpp +++ b/Source/Particles/Resampling/Resampling.cpp @@ -12,7 +12,7 @@ #include #include -Resampling::Resampling (const std::string species_name) +Resampling::Resampling (const std::string& species_name) { const amrex::ParmParse pp_species_name(species_name); std::string resampling_algorithm_string = "leveling_thinning"; // default resampling algorithm diff --git a/Source/Particles/Resampling/ResamplingTrigger.H b/Source/Particles/Resampling/ResamplingTrigger.H index 42c3902c247..538680475a0 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.H +++ b/Source/Particles/Resampling/ResamplingTrigger.H @@ -33,7 +33,7 @@ public: * \brief Constructor of the ResamplingTrigger class. Reads the resampling trigger parameters * from the input file. */ - ResamplingTrigger (std::string species_name); + ResamplingTrigger (const std::string& species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/ResamplingTrigger.cpp b/Source/Particles/Resampling/ResamplingTrigger.cpp index 16a32dd7d15..5efb473fe68 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.cpp +++ b/Source/Particles/Resampling/ResamplingTrigger.cpp @@ -14,7 +14,7 @@ #include -ResamplingTrigger::ResamplingTrigger (const std::string species_name) +ResamplingTrigger::ResamplingTrigger (const std::string& species_name) { const amrex::ParmParse pp_species_name(species_name); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c100cc0bff4..4b01f09e27e 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -467,9 +467,9 @@ protected: //Species can receive a shared pointer to a QED engine (species for //which this is relevant should override these functions) virtual void - set_breit_wheeler_engine_ptr(std::shared_ptr){} + set_breit_wheeler_engine_ptr(const std::shared_ptr&){} virtual void - set_quantum_sync_engine_ptr(std::shared_ptr){} + set_quantum_sync_engine_ptr(const std::shared_ptr&){} int m_qed_breit_wheeler_ele_product; std::string m_qed_breit_wheeler_ele_product_name; diff --git a/Source/Python/callbacks.H b/Source/Python/callbacks.H index f0c98caf636..7ec78782bed 100644 --- a/Source/Python/callbacks.H +++ b/Source/Python/callbacks.H @@ -29,21 +29,21 @@ extern WARPX_EXPORT std::map< std::string, std::function > warpx_callbac /** * \brief Function to install the given name and function in warpx_callback_py_map */ -void InstallPythonCallback ( std::string name, std::function callback ); +void InstallPythonCallback ( const std::string& name, std::function callback ); /** * \brief Function to check if the given name is a key in warpx_callback_py_map */ -bool IsPythonCallbackInstalled ( std::string name ); +bool IsPythonCallbackInstalled ( const std::string& name ); /** * \brief Function to look for and execute Python callbacks */ -void ExecutePythonCallback ( std::string name ); +void ExecutePythonCallback ( const std::string& name ); /** * \brief Function to clear the given callback name from warpx_callback_py_map */ -void ClearPythonCallback ( std::string name ); +void ClearPythonCallback ( const std::string& name ); #endif // WARPX_PY_CALLBACKS_H_ diff --git a/Source/Python/callbacks.cpp b/Source/Python/callbacks.cpp index 930c88e65d1..79f15c62835 100644 --- a/Source/Python/callbacks.cpp +++ b/Source/Python/callbacks.cpp @@ -15,18 +15,18 @@ std::map< std::string, std::function > warpx_callback_py_map; -void InstallPythonCallback ( std::string name, std::function callback ) +void InstallPythonCallback ( const std::string& name, std::function callback ) { - warpx_callback_py_map[name] = callback; + warpx_callback_py_map[name] = std::move(callback); } -bool IsPythonCallbackInstalled ( std::string name ) +bool IsPythonCallbackInstalled ( const std::string& name ) { return (warpx_callback_py_map.count(name) == 1u); } // Execute Python callbacks of the type given by the input string -void ExecutePythonCallback ( std::string name ) +void ExecutePythonCallback ( const std::string& name ) { if ( IsPythonCallbackInstalled(name) ) { WARPX_PROFILE("warpx_py_" + name); @@ -47,7 +47,7 @@ void ExecutePythonCallback ( std::string name ) } } -void ClearPythonCallback ( std::string name ) +void ClearPythonCallback ( const std::string& name ) { warpx_callback_py_map.erase(name); } diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index ad3194edfc3..1de03eb61f0 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -64,7 +64,7 @@ namespace WarpXUtilIO{ * @param[in] data Vector containing binary data to write on disk * return true if it succeeds, false otherwise */ -bool WriteBinaryDataOnFile(std::string filename, const amrex::Vector& data); +bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data); } @@ -116,7 +116,7 @@ namespace WarpXUtilLoadBalance * @param[in] dm the dmap to check * @return consistent whether the grids are consistent or not. */ - bool doCosts (const amrex::LayoutData* cost, amrex::BoxArray ba, + bool doCosts (const amrex::LayoutData* cost, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); } diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 5b917a81fdf..e08a6304f09 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -266,7 +266,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) } namespace WarpXUtilIO{ - bool WriteBinaryDataOnFile(std::string filename, const amrex::Vector& data) + bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data) { std::ofstream of{filename, std::ios::binary}; of.write(data.data(), data.size()); @@ -481,11 +481,11 @@ void ReadBCParams () namespace WarpXUtilLoadBalance { - bool doCosts (const amrex::LayoutData* costs, const amrex::BoxArray ba, + bool doCosts (const amrex::LayoutData* cost, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) { - const bool consistent = costs && (dm == costs->DistributionMap()) && - (ba.CellEqual(costs->boxArray())) && + const bool consistent = cost && (dm == cost->DistributionMap()) && + (ba.CellEqual(cost->boxArray())) && (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers); return consistent; } diff --git a/Source/WarpX.H b/Source/WarpX.H index 1a56445f8dd..0c86a1c9afe 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1050,8 +1050,8 @@ public: * for a specific field (specified by `F_name`) */ void ReadExternalFieldFromFile ( - std::string read_fields_from_path, amrex::MultiFab* mf, - std::string F_name, std::string F_component); + const std::string& read_fields_from_path, amrex::MultiFab* mf, + const std::string& F_name, const std::string& F_component); /** * \brief diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 9c6053bed07..77c06f11cf2 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -96,9 +96,9 @@ computePhi (amrex::Vector const & rho, amrex::Real absolute_tolerance, int const max_iters, int const verbosity, - amrex::Vector const geom, - amrex::Vector const dmap, - amrex::Vector const grids, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index df2287df7f4..d49335723d8 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -89,9 +89,9 @@ computeVectorPotential ( amrex::Vector > co amrex::Real absolute_tolerance, int const max_iters, int const verbosity, - amrex::Vector const geom, - amrex::Vector const dmap, - amrex::Vector const grids, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.H b/Source/ablastr/utils/msg_logger/MsgLogger.H index 01a61be5c80..401432f5dda 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.H +++ b/Source/ablastr/utils/msg_logger/MsgLogger.H @@ -196,7 +196,7 @@ namespace ablastr::utils::msg_logger * * @param[in] msg a Msg struct */ - void record_msg(Msg msg); + void record_msg(const Msg& msg); /** * \brief This function returns a vector containing the recorded messages diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index dde041652da..6537a8f61e5 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -216,7 +216,7 @@ Logger::Logger() : m_io_rank{amrex::ParallelDescriptor::IOProcessorNumber()} {} -void Logger::record_msg(Msg msg) +void Logger::record_msg(const Msg& msg) { m_messages[msg]++; } diff --git a/Source/ablastr/warn_manager/WarnManager.H b/Source/ablastr/warn_manager/WarnManager.H index 737fb8fba73..56ff1bf1a1a 100644 --- a/Source/ablastr/warn_manager/WarnManager.H +++ b/Source/ablastr/warn_manager/WarnManager.H @@ -84,9 +84,9 @@ namespace ablastr::warn_manager * @param[in] priority priority of the warning message ("medium" by default) */ void RecordWarning( - std::string topic, - std::string text, - WarnPriority priority = WarnPriority::medium); + const std::string& topic, + const std::string& text, + const WarnPriority& priority = WarnPriority::medium); /** * \brief This function prints all the warning messages collected on the present MPI rank @@ -225,9 +225,9 @@ namespace ablastr::warn_manager * @param[in] priority priority of the warning message ("medium" by default) */ void WMRecordWarning( - std::string topic, - std::string text, - WarnPriority priority = WarnPriority::medium); + const std::string& topic, + const std::string& text, + const WarnPriority& priority = WarnPriority::medium); } #endif //ABLASTR_WARN_MANAGER_H_ diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index ee052ded299..b1a194b2633 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -55,9 +55,9 @@ WarnManager::WarnManager(): {} void WarnManager::RecordWarning( - std::string topic, - std::string text, - WarnPriority priority) + const std::string& topic, + const std::string& text, + const WarnPriority& priority) { auto msg_priority = abl_msg_logger::Priority::high; if(priority == WarnPriority::low) { @@ -316,9 +316,9 @@ WarnManager& ablastr::warn_manager::GetWMInstance() } void ablastr::warn_manager::WMRecordWarning( - std::string topic, - std::string text, - WarnPriority priority) + const std::string& topic, + const std::string& text, + const WarnPriority& priority) { WarnManager::GetInstance().RecordWarning( topic, text, priority); From e30472d531188c16b99987bce24f9e14dc6f92bd Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 1 Apr 2024 13:26:30 -0700 Subject: [PATCH 038/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4814) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 739684f6c74..259c0d8d351 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - + cd ../amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 8f7da493f24..f0ce221df0c 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c +branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 42274fa30ae..897fcde3cf9 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c +branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index fdc628f3f06..db230573853 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "9e8f8d60eca337e3c14396f06f0a77500289d45c" +set(WarpX_amrex_branch "47347f785f5c26f1f9e65bef8e10f2d383406ef7" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index e2b83469010..8ea26792217 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "bde190184effaf095f14e173df8f61f31ed67308" +set(WarpX_pyamrex_branch "b41b83d1c0f37c66ad0543afa7edaca9e0c82565" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 593d7bf07ca..f74eb50af8c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - +cd amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 6a4ebd9d2c58d11386cdf544ca9ea16c77d05a10 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 1 Apr 2024 22:40:46 +0200 Subject: [PATCH 039/121] Add clang-tidy workflow (#4806) * added instructions and script to run clang-tidy locally * make script executable * remove installation of blaspp and lapackpp from runClangTidy.sh script * improved documentation --- .../developers/run_clang_tidy_locally.rst | 55 ++++++++ Docs/source/developers/workflows.rst | 1 + Tools/Linter/runClangTidy.sh | 117 ++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 Docs/source/developers/run_clang_tidy_locally.rst create mode 100755 Tools/Linter/runClangTidy.sh diff --git a/Docs/source/developers/run_clang_tidy_locally.rst b/Docs/source/developers/run_clang_tidy_locally.rst new file mode 100644 index 00000000000..3f600019fe7 --- /dev/null +++ b/Docs/source/developers/run_clang_tidy_locally.rst @@ -0,0 +1,55 @@ +.. _developers-run_clang_tidy_locally: + +The clang-tidy linter +===================== + +Clang-tidy CI test +------------------ + +WarpX's CI tests include several checks performed with the +`clang-tidy `__ linter +(currently the version 15 of this tool). The complete list of checks +enforced in CI tests can be found in the ``.clang-tidy`` configuration file. + +.. dropdown:: clang-tidy configuration file + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../.clang-tidy + :language: yaml + +Run clang-tidy linter locally +----------------------------- + +We provide a script to run clang-tidy locally. The script can be run as follows, +provided that all the requirements to compile WarpX are met (see `building from source `). +The script generates a simple wrapper to ensure that `clang-tidy` is only applied to WarpX source files +and compiles WarpX in 1D,2D,3D, and RZ using such wrapper. By default WarpX is compiled in single precision +with PSATD solver, QED module, QED table generator and Embedded boundary in order to find more +potential issues with the `clang-tidy` tool. + +Few optional environment variables can be set to tune the behavior of the script: + +* ``WARPX_TOOLS_LINTER_PARALLEL``: sets the number of cores to be used for the compilation +* ``CLANG``, ``CLANGXX``, and ``CLANGTIDY`` : set the version of the compiler and of the linter + +Note: clang v15 is currently used in CI tests. It is therefore recommended to use this version. +Otherwise, a newer version may find issues not currently covered by CI tests (checks are opt-in) +while older versions may not find all the issues. + +.. code-block:: bash + + export WARPX_TOOLS_LINTER_PARALLEL=12 + export CLANG=clang-15 + export CLANGXX=clang++-15 + export CLANGTIDY=clang-tidy-15 + ./Tools/Linter/runClangTidy.sh + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../Tools/Linter/runClangTidy.sh + :language: bash diff --git a/Docs/source/developers/workflows.rst b/Docs/source/developers/workflows.rst index 8b357f5d4da..00279018e9d 100644 --- a/Docs/source/developers/workflows.rst +++ b/Docs/source/developers/workflows.rst @@ -11,3 +11,4 @@ Workflows documentation checksum local_compile + run_clang_tidy_locally diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh new file mode 100755 index 00000000000..39a96cc97d4 --- /dev/null +++ b/Tools/Linter/runClangTidy.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# Copyright 2024 Luca Fedeli +# +# This file is part of WarpX. +# + +# This script is a developer's tool to perform the +# checks done by the clang-tidy CI test locally. +# +# Note: this script is only tested on Linux + +echo "=============================================" +echo +echo "This script is a developer's tool to perform the" +echo "checks done by the clang-tidy CI test locally" +echo "_____________________________________________" + +# Check source dir +REPO_DIR=$(cd $(dirname ${BASH_SOURCE})/../../ && pwd) +echo +echo "Your current source directory is: ${REPO_DIR}" +echo "_____________________________________________" + +# Set number of jobs to use for compilation +PARALLEL="${WARPX_TOOLS_LINTER_PARALLEL:-4}" +echo +echo "${PARALLEL} jobs will be used for compilation." +echo "This can be overridden by setting the environment" +echo "variable WARPX_TOOLS_LINTER_PARALLEL, e.g.: " +echo +echo "$ export WARPX_TOOLS_LINTER_PARALLEL=8" +echo "$ ./Tools/Linter/runClangTidy.sh" +echo "_____________________________________________" + +# Check clang version +export CC="${CLANG:-"clang"}" +export CXX="${CLANGXX:-"clang++"}" +export CTIDY="${CLANGTIDY:-"clang-tidy"}" +echo +echo "The following versions of the clang compiler and" +echo "of the clang-tidy linter will be used:" +echo +echo "clang version:" +which ${CC} +${CC} --version +echo +echo "clang++ version:" +which ${CXX} +${CXX} --version +echo +echo "clang-tidy version:" +which ${CTIDY} +${CTIDY} --version +echo +echo "This can be overridden by setting the environment" +echo "variables CLANG, CLANGXX, and CLANGTIDY e.g.: " +echo "$ export CLANG=clang-15" +echo "$ export CLANGXX=clang++-15" +echo "$ export CTIDCLANGTIDYY=clang-tidy-15" +echo "$ ./Tools/Linter/runClangTidy.sh" +echo +echo "******************************************************" +echo "* Warning: clang v15 is currently used in CI tests. *" +echo "* It is therefore recommended to use this version. *" +echo "* Otherwise, a newer version may find issues not *" +echo "* currently covered by CI tests while older versions *" +echo "* may not find all the issues. *" +echo "******************************************************" +echo "_____________________________________________" + +# Prepare clang-tidy wrapper +echo +echo "Prepare clang-tidy wrapper" +echo "The following wrapper ensures that only source files" +echo "in WarpX/Source/* are actually processed by clang-tidy" +echo +cat > ${REPO_DIR}/clang_tidy_wrapper << EOF +#!/bin/bash +REGEX="[a-z_A-Z0-9\/]*WarpX\/Source[a-z_A-Z0-9\/]+.cpp" +if [[ \$4 =~ \$REGEX ]];then + ${CTIDY} \$@ +fi +EOF +chmod +x ${REPO_DIR}/clang_tidy_wrapper +echo "clang_tidy_wrapper: " +cat ${REPO_DIR}/clang_tidy_wrapper +echo "_____________________________________________" + +# Compile Warpx using clang-tidy +echo +echo "*******************************************" +echo "* Compile Warpx using clang-tidy *" +echo "* Please ensure that all the dependencies *" +echo "* required to compile WarpX are met *" +echo "*******************************************" +echo + +rm -rf ${REPO_DIR}/build_clang_tidy + +cmake -S ${REPO_DIR} -B ${REPO_DIR}/build_clang_tidy \ + -DCMAKE_CXX_CLANG_TIDY="${REPO_DIR}/clang_tidy_wrapper;--system-headers=0;--config-file=${REPO_DIR}/.clang-tidy" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DWarpX_DIMS="1;2;3;RZ" \ + -DWarpX_MPI=ON \ + -DWarpX_COMPUTE=OMP \ + -DWarpX_PSATD=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE + +cmake --build ${REPO_DIR}/build_clang_tidy -j ${PARALLEL} 2> ${REPO_DIR}/build_clang_tidy/clang-tidy.log + +cat ${REPO_DIR}/build_clang_tidy/clang-tidy.log +echo +echo "=============================================" From 0466145fc0edf14d23ec5acd3845e52f8c30127e Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 1 Apr 2024 23:07:31 +0200 Subject: [PATCH 040/121] Clang tidy CI test: add several clang-analyzer-* checks to clang tidy CI test (#4684) * Clang tidy CI test: add several clang-analyzer-* checks to clang tidy CI test * address issues found with clang-tidy * fix bug * fix spaces * simplify * avoid using potentially confusing v[counter++] syntax * disable clang-analyzer-optin.mpi.MPI-Checker * make thread_required constexpr in mpi_init * make thread_required constexpr in mpi_init * disable clang-analyzer-optin.performance.Padding * start using get_field_pointer * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * introduce AMREX_ASSUME * update amrex * using get_field_pointer more * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * use new function get_field_pointer * use new functions to get fields and field pointers everywhere * fix bug * bugfixing * add missing [[nodiscard]] and remove redundant [[nodiscard]] * address issue related to auto* const found with clang-tidy * fix bug * fix issue found with clang tidy * fixed bug * fix bug --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .clang-tidy | 5 + Source/BoundaryConditions/PML.cpp | 37 +++-- Source/BoundaryConditions/PML_RZ.cpp | 8 +- .../WarpXFieldBoundaries.cpp | 48 +++--- Source/Diagnostics/BTDiagnostics.cpp | 40 ++--- .../BackTransformFunctor.cpp | 22 ++- .../ComputeDiagFunctors/CellCenterFunctor.cpp | 10 +- .../ComputeDiagFunctors/JFunctor.cpp | 17 +- .../ComputeDiagFunctors/JdispFunctor.cpp | 23 +-- .../FlushFormats/FlushFormatCheckpoint.cpp | 60 +++---- .../FlushFormats/FlushFormatPlotfile.cpp | 76 ++++----- Source/Diagnostics/FullDiagnostics.cpp | 127 +++++++-------- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 6 +- .../ReducedDiags/ColliderRelevant.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 12 +- .../ReducedDiags/FieldMomentum.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldReduction.H | 18 +- .../ReducedDiags/LoadBalanceCosts.cpp | 2 +- .../ReducedDiags/ParticleExtrema.cpp | 12 +- Source/Diagnostics/SliceDiagnostic.cpp | 12 +- Source/FieldSolver/ElectrostaticSolver.cpp | 12 +- .../HybridPICModel/HybridPICModel.cpp | 26 +-- .../MacroscopicProperties.cpp | 6 +- Source/Parallelization/GuardCellManager.cpp | 2 + Source/Parallelization/WarpXComm_K.H | 34 ++-- Source/Particles/MultiParticleContainer.cpp | 12 +- .../Particles/PhysicalParticleContainer.cpp | 10 +- Source/WarpX.H | 154 ++++++++++-------- Source/WarpX.cpp | 124 ++++++++++++++ .../ablastr/parallelization/MPIInitHelpers.H | 2 +- .../parallelization/MPIInitHelpers.cpp | 20 +-- 33 files changed, 574 insertions(+), 411 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 5d2422af214..04d1419c5c7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -7,6 +7,11 @@ Checks: ' -bugprone-unchecked-optional-access, cert-*, -cert-err58-cpp, + clang-analyzer-*, + -clang-analyzer-optin.performance.Padding, + -clang-analyzer-optin.mpi.MPI-Checker, + -clang-analyzer-osx.*, + -clang-analyzer-optin.osx.*, clang-diagnostic-*, cppcoreguidelines-*, -cppcoreguidelines-avoid-c-arrays, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 805b4fec181..0b2806b3151 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -678,23 +678,23 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const int ncompe = (m_dive_cleaning) ? 3 : 2; const int ncompb = (m_divb_cleaning) ? 3 : 2; - const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); - const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); @@ -812,16 +812,17 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } else { cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,2).ixType().toIntVect()); + + const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_cp[0], cba_Ex, cdm, ncompe, nge, lev, "pml_E_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[1], cba_Ey, cdm, ncompe, nge, lev, "pml_E_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[2], cba_Ez, cdm, ncompe, nge, lev, "pml_E_cp[z]", 0.0_rt); - const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,2).ixType().toIntVect()); + const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_cp[0], cba_Bx, cdm, ncompb, ngb, lev, "pml_B_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[1], cba_By, cdm, ncompb, ngb, lev, "pml_B_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[2], cba_Bz, cdm, ncompb, ngb, lev, "pml_B_cp[z]", 0.0_rt); @@ -842,9 +843,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,2).ixType().toIntVect()); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 7eb011a6bb2..5eb30f43cca 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -41,15 +41,15 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist m_geom(geom) { - const amrex::MultiFab & Er_fp = WarpX::GetInstance().getEfield_fp(lev,0); - const amrex::MultiFab & Et_fp = WarpX::GetInstance().getEfield_fp(lev,1); + const amrex::MultiFab & Er_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,0); + const amrex::MultiFab & Et_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,1); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); - const amrex::MultiFab & Br_fp = WarpX::GetInstance().getBfield_fp(lev,0); - const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getBfield_fp(lev,1); + const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); + const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 4844585cc90..ebf3d1e39eb 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -22,18 +22,18 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) { if (PEC::isAnyBoundaryPEC()) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoEfield( { get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2) }, lev, patch_type); if (WarpX::isAnyBoundaryPML()) { // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( pml[lev]->GetE_fp(), lev, patch_type, split_pml_field); } } else { - PEC::ApplyPECtoEfield( { get_pointer_Efield_cp(lev, 0), - get_pointer_Efield_cp(lev, 1), - get_pointer_Efield_cp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2) }, lev, patch_type); if (WarpX::isAnyBoundaryPML()) { // apply pec on split E-fields in PML region const bool split_pml_field = true; @@ -44,13 +44,13 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2), lev); } else { - ApplyFieldBoundaryOnAxis(get_pointer_Efield_cp(lev, 0), - get_pointer_Efield_cp(lev, 1), - get_pointer_Efield_cp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2), lev); } #endif } @@ -59,13 +59,13 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d { if (PEC::isAnyBoundaryPEC()) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoBfield( { get_pointer_Bfield_fp(lev, 0), - get_pointer_Bfield_fp(lev, 1), - get_pointer_Bfield_fp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2) }, lev, patch_type); } else { - PEC::ApplyPECtoBfield( { get_pointer_Bfield_cp(lev, 0), - get_pointer_Bfield_cp(lev, 1), - get_pointer_Bfield_cp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2) }, lev, patch_type); } } @@ -92,13 +92,13 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(get_pointer_Bfield_fp(lev, 0), - get_pointer_Bfield_fp(lev, 1), - get_pointer_Bfield_fp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2), lev); } else { - ApplyFieldBoundaryOnAxis(get_pointer_Bfield_cp(lev, 0), - get_pointer_Bfield_cp(lev, 1), - get_pointer_Bfield_cp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2), lev); } #endif } diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 2422d1324af..40a76eac946 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -565,23 +565,23 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "By" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -596,7 +596,7 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -655,7 +655,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point @@ -681,23 +681,23 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_cell_center_functors_at_lev_size = static_cast(m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index c4809df11a3..199b785096d 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -58,15 +59,22 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const const bool interpolate = true; std::unique_ptr< amrex::MultiFab > slice = nullptr; const int scomp = 0; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + // Generate slice of the cell-centered multifab containing boosted-frame field-data // at current z-boost location for the ith buffer - slice = amrex::get_slice_data(moving_window_dir, - m_current_z_boost[i_buffer], - *m_mf_src, - geom, - scomp, - m_mf_src->nComp(), - interpolate); + slice = amrex::get_slice_data( + moving_window_dir, + m_current_z_boost[i_buffer], + *m_mf_src, + geom, + scomp, + m_mf_src->nComp(), + interpolate); + + // Perform in-place Lorentz-transform of all the fields stored in the slice. LorentzTransformZ( *slice, gamma_boost, beta_boost); diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp index fe503713141..8c2af223304 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp @@ -3,6 +3,7 @@ #include "WarpX.H" #include +#include #include #include @@ -17,6 +18,11 @@ void CellCenterFunctor::operator()(amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { auto& warpx = WarpX::GetInstance(); - InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), - m_convertRZmodes2cartesian); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + + InterpolateMFForDiag( + mf_dst, *m_mf_src, dcomp, + warpx.DistributionMap(m_lev),m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 9938cf095bd..a8fd142815a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -10,6 +10,7 @@ #include "WarpX.H" #include +#include #include #include @@ -27,7 +28,7 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ { auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* m_mf_src = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) @@ -36,16 +37,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getField(FieldType::current_fp, m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getField(FieldType::current_fp, m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getField(FieldType::current_fp, m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); @@ -60,6 +61,10 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ } } - InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), - m_convertRZmodes2cartesian); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + + InterpolateMFForDiag( + mf_dst, *m_mf_src, dcomp, + warpx.DistributionMap(m_lev), m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index 373004d4633..b73931814b6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -10,6 +10,7 @@ #include "Particles/MultiParticleContainer.H" #include +#include #include #include @@ -28,24 +29,26 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); - amrex::MultiFab* mf_curlB; - if (hybrid_pic_model) { - /** pointer to current calculated from Ampere's Law (Jamp) multifab */ - mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); - } else { + amrex::MultiFab* mf_j = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, + "Displacement current diagnostic is only implemented for the HybridPICModel."); + AMREX_ASSUME(hybrid_pic_model != nullptr); + + /** pointer to current calculated from Ampere's Law (Jamp) multifab */ + amrex::MultiFab* mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir);; + + //if (!hybrid_pic_model) { // To finish this implementation, we need to implement a method to // calculate (∇ x B). - mf_curlB = nullptr; // Remove when curlB implemented - WARPX_ABORT_WITH_MESSAGE( - "Displacement current diagnostic is only implemented for the HybridPICModel."); // Skeleton for future implementation for solvers other than HybridPIC. // Get curlB multifab // Divide curlB multifab by mu0 to get units of current // mf_curlB->mult(1.0/PhysConsts::mu0) - } + //} + // A Jdisp multifab is generated to hold displacement current. amrex::MultiFab Jdisp( mf_j->boxArray(), mf_j->DistributionMap(), 1, mf_j->nGrowVect() ); Jdisp.setVal(0); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index b083e60529f..653ac0c26ba 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -62,85 +62,85 @@ FlushFormatCheckpoint::WriteToFile ( for (int lev = 0; lev < nlev; ++lev) { - VisMF::Write(warpx.getEfield_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_fp")); - VisMF::Write(warpx.getEfield_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); - VisMF::Write(warpx.getEfield_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getEfield_avg_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_fp")); - VisMF::Write(warpx.getEfield_avg_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_fp")); - VisMF::Write(warpx.getEfield_avg_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_fp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getcurrent_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(warpx.getcurrent_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(warpx.getcurrent_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Write(warpx.getEfield_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_cp")); - VisMF::Write(warpx.getEfield_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_cp")); - VisMF::Write(warpx.getEfield_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getEfield_avg_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_cp")); - VisMF::Write(warpx.getEfield_avg_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_cp")); - VisMF::Write(warpx.getEfield_avg_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_cp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getcurrent_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_cp")); - VisMF::Write(warpx.getcurrent_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_cp")); - VisMF::Write(warpx.getcurrent_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_cp")); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 555ffb123e7..7b10edf0d2b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -562,84 +562,84 @@ FlushFormatPlotfile::WriteAllRawFields( // Auxiliary patch - WriteRawMF( warpx.getEfield(lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield(lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield(lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch - WriteRawMF( warpx.getEfield_fp(lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield_fp(lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield_fp(lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); - if (warpx.get_pointer_F_fp(lev)) + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); + if (warpx.isFieldInitialized(FieldType::F_fp, lev)) { - WriteRawMF(warpx.getF_fp(lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); + WriteRawMF(warpx.getField(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); } - if (warpx.get_pointer_rho_fp(lev)) + if (warpx.isFieldInitialized(FieldType::rho_fp, lev)) { // rho_fp will have either ncomps or 2*ncomps (2 being the old and new). When 2, return the new so // there is time synchronization. - const int nstart = warpx.getrho_fp(lev).nComp() - WarpX::ncomps; - const MultiFab rho_new(warpx.getrho_fp(lev), amrex::make_alias, nstart, WarpX::ncomps); + const int nstart = warpx.getField(FieldType::rho_fp, lev).nComp() - WarpX::ncomps; + const MultiFab rho_new(warpx.getField(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } - if (warpx.get_pointer_phi_fp(lev) != nullptr) { - WriteRawMF(warpx.getphi_fp(lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + if (warpx.isFieldInitialized(FieldType::phi_fp, lev)) { + WriteRawMF(warpx.getField(FieldType::phi_fp, lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); } // Averaged fields on fine patch if (WarpX::fft_do_time_averaging) { - WriteRawMF(warpx.getEfield_avg_fp(lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, "Ex_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getEfield_avg_fp(lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, "Ey_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getEfield_avg_fp(lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, "Ez_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, "Bx_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, "By_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, "Bz_avg_fp", lev, plot_raw_fields_guards); } // Coarse path if (lev > 0) { WriteCoarseVector( "E", - warpx.get_pointer_Efield_cp(lev, 0), warpx.get_pointer_Efield_cp(lev, 1), warpx.get_pointer_Efield_cp(lev, 2), - warpx.get_pointer_Efield_fp(lev, 0), warpx.get_pointer_Efield_fp(lev, 1), warpx.get_pointer_Efield_fp(lev, 2), + warpx.getFieldPointer(FieldType::Efield_cp, lev, 0), warpx.getFieldPointer(FieldType::Efield_cp, lev, 1), warpx.getFieldPointer(FieldType::Efield_cp, lev, 2), + warpx.getFieldPointer(FieldType::Efield_fp, lev, 0), warpx.getFieldPointer(FieldType::Efield_fp, lev, 1), warpx.getFieldPointer(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", - warpx.get_pointer_Bfield_cp(lev, 0), warpx.get_pointer_Bfield_cp(lev, 1), warpx.get_pointer_Bfield_cp(lev, 2), - warpx.get_pointer_Bfield_fp(lev, 0), warpx.get_pointer_Bfield_fp(lev, 1), warpx.get_pointer_Bfield_fp(lev, 2), + warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), + warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.get_pointer_current_cp(lev, 0), warpx.get_pointer_current_cp(lev, 1), warpx.get_pointer_current_cp(lev, 2), - warpx.get_pointer_current_fp(lev, 0), warpx.get_pointer_current_fp(lev, 1), warpx.get_pointer_current_fp(lev, 2), + warpx.getFieldPointer(FieldType::current_cp, lev, 0), warpx.getFieldPointer(FieldType::current_cp, lev, 1), warpx.getFieldPointer(FieldType::current_cp, lev, 2), + warpx.getFieldPointer(FieldType::current_fp, lev, 0), warpx.getFieldPointer(FieldType::current_fp, lev, 1), warpx.getFieldPointer(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); - if (warpx.get_pointer_F_fp(lev) && warpx.get_pointer_F_cp(lev)) + if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) { - WriteCoarseScalar("F", warpx.get_pointer_F_cp(lev), warpx.get_pointer_F_fp(lev), + WriteCoarseScalar("F", warpx.getFieldPointer(FieldType::F_cp, lev), warpx.getFieldPointer(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } - if (warpx.get_pointer_rho_fp(lev) && warpx.get_pointer_rho_cp(lev)) + if (warpx.isFieldInitialized(FieldType::rho_fp, lev) && warpx.isFieldInitialized(FieldType::rho_cp, lev)) { // Use the component 1 of `rho_cp`, i.e. rho_new for time synchronization - WriteCoarseScalar("rho", warpx.get_pointer_rho_cp(lev), warpx.get_pointer_rho_fp(lev), + WriteCoarseScalar("rho", warpx.getFieldPointer(FieldType::rho_cp, lev), warpx.getFieldPointer(FieldType::rho_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 1); } } diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index bf4cd3f95fc..63683690483 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -170,15 +170,15 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -211,37 +211,37 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Er"), ncomp); } } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Et"), ncomp); } } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Ez"), ncomp); } } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Br"), ncomp); } } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bt"), ncomp); } } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bz"), ncomp); @@ -301,19 +301,19 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } i++; } else if ( m_varnames_fields[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_G_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); } } else if ( m_varnames_fields[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_phi_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("phi"), ncomp); @@ -329,14 +329,16 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) m_varnames.push_back(std::string("part_per_grid")); } } else if ( m_varnames_fields[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio, - false, ncomp); + m_all_field_functors[lev][comp] = std::make_unique( + warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), + lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divB"), ncomp); } } else if ( m_varnames_fields[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio, - false, ncomp); + m_all_field_functors[lev][comp] = std::make_unique( + warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divE"), ncomp); } @@ -382,15 +384,15 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); } // Check if divE is requested @@ -409,8 +411,6 @@ FullDiagnostics::AddRZModesToDiags (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; - // First index of m_all_field_functors[lev] where RZ modes are stored - auto icomp =static_cast(m_all_field_functors[0].size()); const std::array coord {"r", "theta", "z"}; // Er, Etheta, Ez, Br, Btheta, Bz, jr, jtheta, jz @@ -422,48 +422,45 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (rho_requested) { n_new_fields += 1; } - m_all_field_functors[lev].resize( m_all_field_functors[0].size() + n_new_fields ); + m_all_field_functors[lev].reserve( m_all_field_functors[0].size() + n_new_fields ); // E for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Efield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.get_pointer_Efield_aux(0, 0)->nComp()); - icomp += 1; + warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp()); } // B for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Bfield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.get_pointer_Bfield_aux(0, 0)->nComp()); - icomp += 1; + warpx.getFieldPointer(FieldType::Bfield_aux, 0, 0)->nComp()); } // j for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; - icomp += 1; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.get_pointer_current_fp(0, 0)->nComp()); + warpx.getFieldPointer(FieldType::current_fp, 0, 0)->nComp()); } // divE if (divE_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, - m_crse_ratio, false, ncomp_multimodefab); - icomp += 1; + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); } // rho if (rho_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp_multimodefab); - icomp += 1; + m_all_field_functors[lev].push_back(std::make_unique( + lev, m_crse_ratio, true, -1, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("rho"), ncomp_multimodefab); } // Sum the number of components in input vector m_all_field_functors @@ -649,16 +646,16 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; } else if ( m_varnames[comp] == "jz_displacement" ) { m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); @@ -667,31 +664,31 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_G_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_phi_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_grid" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Efield_aux, lev), lev, m_crse_ratio); } else { #ifdef WARPX_DIM_RZ if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jr" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -703,24 +700,22 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if (m_varnames[comp] == "jt_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); - } - else { - std::cout << "Error on component " << m_varnames[comp] << std::endl; + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); + } else { WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } #else // Valid transverse fields in Cartesian coordinates if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jx" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -732,9 +727,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); } else { std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 1398072a84d..7e66061fa48 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -93,9 +93,9 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev - const amrex::MultiFab & Ex = warpx.getEfield_fp(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield_fp(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield_fp(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_fp, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_fp, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_fp, lev,2); // get EB structures amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 3229f10c89e..88dd0c0922b 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -441,12 +441,12 @@ void ColliderRelevant::ComputeDiags (int step) // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::MultiFab & Ex = warpx.getEfield(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield(lev,2); - const amrex::MultiFab & Bx = warpx.getBfield(lev,0); - const amrex::MultiFab & By = warpx.getBfield(lev,1); - const amrex::MultiFab & Bz = warpx.getBfield(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 1271e285c6c..65b33341d1a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -89,12 +89,12 @@ void FieldEnergy::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // get cell size Geometry const & geom = warpx.Geom(lev); diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 8c66404056b..80979cd845e 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -114,12 +114,12 @@ void FieldMaximum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); constexpr int noutputs = 8; // max of Ex,Ey,Ez,|E|,Bx,By,Bz and |B| constexpr int index_Ex = 0; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 3b3b7463593..901748bd6a9 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -106,12 +106,12 @@ void FieldMomentum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // Get MultiFab data at given refinement level - const amrex::MultiFab & Ex = warpx.getEfield(lev, 0); - const amrex::MultiFab & Ey = warpx.getEfield(lev, 1); - const amrex::MultiFab & Ez = warpx.getEfield(lev, 2); - const amrex::MultiFab & Bx = warpx.getBfield(lev, 0); - const amrex::MultiFab & By = warpx.getBfield(lev, 1); - const amrex::MultiFab & Bz = warpx.getBfield(lev, 2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev, 0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev, 1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev, 2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev, 1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); // Cell-centered index type const amrex::GpuArray cc{0,0,0}; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index f0780622d54..4be5a928d96 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -391,12 +391,12 @@ void FieldProbe::ComputeDiags (int step) } // get MultiFab data at lev - const amrex::MultiFab &Ex = warpx.getEfield(lev, 0); - const amrex::MultiFab &Ey = warpx.getEfield(lev, 1); - const amrex::MultiFab &Ez = warpx.getEfield(lev, 2); - const amrex::MultiFab &Bx = warpx.getBfield(lev, 0); - const amrex::MultiFab &By = warpx.getBfield(lev, 1); - const amrex::MultiFab &Bz = warpx.getBfield(lev, 2); + const amrex::MultiFab &Ex = warpx.getField(FieldType::Efield_aux, lev, 0); + const amrex::MultiFab &Ey = warpx.getField(FieldType::Efield_aux, lev, 1); + const amrex::MultiFab &Ez = warpx.getField(FieldType::Efield_aux, lev, 2); + const amrex::MultiFab &Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); + const amrex::MultiFab &By = warpx.getField(FieldType::Bfield_aux, lev, 1); + const amrex::MultiFab &Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); /* * Prepare interpolation of field components to probe_position diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index 940222867b1..a2813bf51e8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -99,15 +99,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getEfield(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield(lev,2); - const amrex::MultiFab & Bx = warpx.getBfield(lev,0); - const amrex::MultiFab & By = warpx.getBfield(lev,1); - const amrex::MultiFab & Bz = warpx.getBfield(lev,2); - const amrex::MultiFab & jx = warpx.getcurrent_fp(lev,0); - const amrex::MultiFab & jy = warpx.getcurrent_fp(lev,1); - const amrex::MultiFab & jz = warpx.getcurrent_fp(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & jx = warpx.getField(FieldType::current_fp, lev,0); + const amrex::MultiFab & jy = warpx.getField(FieldType::current_fp, lev,1); + const amrex::MultiFab & jz = warpx.getField(FieldType::current_fp, lev,2); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 8af2c66711c..e5e8a7adc07 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -125,7 +125,7 @@ void LoadBalanceCosts::ComputeDiags (int step) for (int lev = 0; lev < nLevels; ++lev) { const amrex::DistributionMapping& dm = warpx.DistributionMap(lev); - const MultiFab & Ex = warpx.getEfield(lev,0); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); for (MFIter mfi(Ex, false); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 0414d9ce869..79b66a92e66 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -373,12 +373,12 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); const GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index f373e78c9b2..aaadac52610 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -195,27 +195,27 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio); } - if( SliceType == WarpX::GetInstance().getEfield(0,0).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,0).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getEfield(0,1).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,1).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getEfield(0,2).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,2).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } - if( SliceType == WarpX::GetInstance().getBfield(0,0).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,0).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getBfield(0,1).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,1).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getBfield(0,2).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,2).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index f1bda102c8b..755c9d6e2a0 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -320,18 +320,18 @@ WarpX::computePhi (const amrex::Vector >& rho, e_field.push_back( # if defined(WARPX_DIM_1D_Z) amrex::Array{ - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 2) } # elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array{ - get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 2) } # elif defined(WARPX_DIM_3D) amrex::Array{ - get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2) } # endif ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index e8330242fe8..228201886eb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -149,15 +149,15 @@ void HybridPICModel::InitData () auto & warpx = WarpX::GetInstance(); // Get the grid staggering of the fields involved in calculating E - amrex::IntVect Jx_stag = warpx.getcurrent_fp(0,0).ixType().toIntVect(); - amrex::IntVect Jy_stag = warpx.getcurrent_fp(0,1).ixType().toIntVect(); - amrex::IntVect Jz_stag = warpx.getcurrent_fp(0,2).ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.getBfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect By_stag = warpx.getBfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.getBfield_fp(0,2).ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getEfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getEfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getEfield_fp(0,2).ixType().toIntVect(); + amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); // Check that the grid types are appropriate const bool appropriate_grids = ( @@ -224,9 +224,9 @@ void HybridPICModel::InitData () for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { #ifdef AMREX_USE_EB - auto& edge_lengths_x = warpx.getedgelengths(lev, 0); - auto& edge_lengths_y = warpx.getedgelengths(lev, 1); - auto& edge_lengths_z = warpx.getedgelengths(lev, 2); + auto& edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); + auto& edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); + auto& edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); const auto edge_lengths = std::array< std::unique_ptr, 3 >{ std::make_unique( @@ -499,7 +499,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) // charge density. if (a_dt_type == DtType::Full) { FillElectronPressureMF( - electron_pressure_fp[lev], warpx.get_pointer_rho_fp(lev) + electron_pressure_fp[lev], warpx.getFieldPointer(FieldType::rho_fp, lev) ); } else { FillElectronPressureMF( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 965db68a558..c60a0b4c4d5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -180,9 +180,9 @@ MacroscopicProperties::InitData () amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getEfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getEfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getEfield_fp(0,2).ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { sigma_IndexType[idim] = sigma_stag[idim]; diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index dc7800f490a..28157e09d8c 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -120,9 +120,11 @@ guardCellManager::Init ( #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) ng_alloc_EB = IntVect(ngx,ngz); ng_alloc_J = IntVect(ngJx,ngJz); + amrex::ignore_unused(ngy, ngJy); #elif defined(WARPX_DIM_1D_Z) ng_alloc_EB = IntVect(ngz); ng_alloc_J = IntVect(ngJz); + amrex::ignore_unused(ngx, ngJx, ngy, ngJy); #endif // TODO Adding one cell for rho should not be necessary, given that the number of guard cells diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index 4cec33bc84f..a2b8fe38ed4 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -179,10 +179,6 @@ void warpx_interp (int j, int k, int l, amrex::Real fine = 0.0_rt; amrex::Real coarse = 0.0_rt; - amrex::Real wj; - amrex::Real wk; - amrex::Real wl; - // 1) Interpolation from coarse nodal to fine nodal nj = 2; @@ -197,20 +193,18 @@ void warpx_interp (int j, int k, int l, nl = 2; #endif - wj = 1.0_rt; - wk = 1.0_rt; - wl = 1.0_rt; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); + auto c = arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + c *= (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); #if (AMREX_SPACEDIM >= 2) - wk = (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); + c *= (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); #endif #if (AMREX_SPACEDIM == 3) - wl = (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); + c *= (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); #endif - tmp += wj * wk * wl * arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + tmp += c; } } } @@ -237,20 +231,18 @@ void warpx_interp (int j, int k, int l, kc = amrex::coarsen(kn, rk); lc = amrex::coarsen(ln, rl); - wj = 1.0_rt; - wk = 1.0_rt; - wl = 1.0_rt; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); + auto c = arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + c *= (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); #if (AMREX_SPACEDIM >= 2) - wk = (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); + c *= (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); #endif #if (AMREX_SPACEDIM == 3) - wl = (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); + c *= (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); #endif - coarse += wj * wk * wl * arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + coarse += c; } } } @@ -284,13 +276,11 @@ void warpx_interp (int j, int k, int l, for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = 1.0_rt / static_cast(nj); - wk = 1.0_rt / static_cast(nk); - wl = 1.0_rt / static_cast(nl); - fine += wj * wk * wl * arr_fine_zeropad(jm+jj,km+kk,lm+ll); + fine += arr_fine_zeropad(jm+jj,km+kk,lm+ll); } } } + fine = fine/static_cast(nj*nk*nl); // Final result arr_aux(j,k,l) = tmp + (fine - coarse); diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index b70844a8d15..97c51c6350b 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -1344,12 +1344,12 @@ MultiParticleContainer::doQEDSchwinger () pc_product_ele->defineAllParticleTiles(); pc_product_pos->defineAllParticleTiles(); - const MultiFab & Ex = warpx.getEfield(level_0,0); - const MultiFab & Ey = warpx.getEfield(level_0,1); - const MultiFab & Ez = warpx.getEfield(level_0,2); - const MultiFab & Bx = warpx.getBfield(level_0,0); - const MultiFab & By = warpx.getBfield(level_0,1); - const MultiFab & Bz = warpx.getBfield(level_0,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, level_0,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, level_0,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, level_0,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, level_0,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, level_0,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, level_0,2); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 9923d93a907..4b2d73c7d6a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2867,11 +2867,11 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, // Using this version of ParallelFor with compile time options // improves performance when qed or external EB are not used by reducing // register pressure. - amrex::ParallelFor(TypeList, - CompileTimeOptions>{}, - {exteb_runtime_flag, qed_runtime_flag}, - np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, - auto qed_control) + amrex::ParallelFor( + TypeList, CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, + [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, auto qed_control) { amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); diff --git a/Source/WarpX.H b/Source/WarpX.H index 0c86a1c9afe..3be95f47865 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -69,12 +69,11 @@ #include #include #include +#include #include #include #include #include -#include - enum struct PatchType : int { @@ -82,6 +81,33 @@ enum struct PatchType : int coarse }; +enum struct FieldType : int +{ + Efield_aux, + Bfield_aux, + Efield_fp, + Bfield_fp, + current_fp, + current_fp_nodal, + rho_fp, + F_fp, + G_fp, + phi_fp, + vector_potential_fp, + Efield_cp, + Bfield_cp, + current_cp, + rho_cp, + F_cp, + G_cp, + edge_lengths, + face_areas, + Efield_avg_fp, + Bfield_avg_fp, + Efield_avg_cp, + Bfield_avg_cp +}; + class WARPX_EXPORT WarpX : public amrex::AmrCore { @@ -463,72 +489,56 @@ public: static std::map multifab_map; static std::map imultifab_map; - [[nodiscard]] std::array - get_array_Bfield_aux (const int lev) const { - return { - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get() - }; - } + /** + * \brief + * Check if a field is initialized. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return true if the field is initialized, false otherwise + */ + [[nodiscard]] bool + isFieldInitialized (FieldType field_type, int lev, int direction = 0) const; + + /** + * \brief + * Get a pointer to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointer (FieldType field_type, int lev, int direction = 0) const; + /** + * \brief + * For vector fields, get an array of three pointers to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * + * \return an array of three pointers amrex::MultiFab* containing the field data + */ [[nodiscard]] std::array - get_array_Efield_aux (const int lev) const { - return { - Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get() - }; - } + getFieldPointerArray (FieldType field_type, int lev) const; - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_aux (int lev, int direction) const { return Efield_aux[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_aux (int lev, int direction) const { return Bfield_aux[lev][direction].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_fp (int lev, int direction) const { return Efield_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_fp (int lev, int direction) const { return Bfield_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_fp (int lev, int direction) const { return current_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_fp_nodal (int lev, int direction) const { return current_fp_nodal[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_rho_fp (int lev) const { return rho_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_F_fp (int lev) const { return F_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_G_fp (int lev) const { return G_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_phi_fp (int lev) const { return phi_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_vector_potential_fp (int lev, int direction) const { return vector_potential_fp_nodal[lev][direction].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_cp (int lev, int direction) const { return Efield_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_cp (int lev, int direction) const { return Bfield_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_cp (int lev, int direction) const { return current_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_rho_cp (int lev) const { return rho_cp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_F_cp (int lev) const { return F_cp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_G_cp (int lev) const { return G_cp[lev].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_edge_lengths (int lev, int direction) const { return m_edge_lengths[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_face_areas (int lev, int direction) const { return m_face_areas[lev][direction].get(); } - - const amrex::MultiFab& getEfield (int lev, int direction) {return *Efield_aux[lev][direction];} - const amrex::MultiFab& getBfield (int lev, int direction) {return *Bfield_aux[lev][direction];} - - const amrex::MultiFab& getcurrent_cp (int lev, int direction) {return *current_cp[lev][direction];} - const amrex::MultiFab& getEfield_cp (int lev, int direction) {return *Efield_cp[lev][direction];} - const amrex::MultiFab& getBfield_cp (int lev, int direction) {return *Bfield_cp[lev][direction];} - const amrex::MultiFab& getrho_cp (int lev) {return *rho_cp[lev];} - const amrex::MultiFab& getF_cp (int lev) {return *F_cp[lev];} - const amrex::MultiFab& getG_cp (int lev) {return *G_cp[lev];} - - const amrex::MultiFab& getcurrent_fp (int lev, int direction) {return *current_fp[lev][direction];} - const amrex::MultiFab& getEfield_fp (int lev, int direction) {return *Efield_fp[lev][direction];} - const amrex::MultiFab& getBfield_fp (int lev, int direction) {return *Bfield_fp[lev][direction];} - const amrex::MultiFab& getrho_fp (int lev) {return *rho_fp[lev];} - const amrex::MultiFab& getphi_fp (int lev) {return *phi_fp[lev];} - const amrex::MultiFab& getF_fp (int lev) {return *F_fp[lev];} - const amrex::MultiFab& getG_fp (int lev) {return *G_fp[lev];} - - const amrex::MultiFab& getEfield_avg_fp (int lev, int direction) {return *Efield_avg_fp[lev][direction];} - const amrex::MultiFab& getBfield_avg_fp (int lev, int direction) {return *Bfield_avg_fp[lev][direction];} - const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];} - const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} - - const amrex::MultiFab& getedgelengths (int lev, int direction) {return *m_edge_lengths[lev][direction];} - const amrex::MultiFab& getfaceareas (int lev, int direction) {return *m_face_areas[lev][direction];} + /** + * \brief + * Get a constant reference to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return a constant refernce to an amrex::MultiFab containing the field data + */ + [[nodiscard]] const amrex::MultiFab& + getField(FieldType field_type, int lev, int direction = 0) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -1586,6 +1596,20 @@ private: amrex::Vector, 3 > > current_buf; amrex::Vector > charge_buf; + /** + * \brief + * Get a pointer to the field data. Does not check if the pointer + * is not nullptr. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointerUnchecked (FieldType field_type, int lev, int direction = 0) const; + // PML int do_pml = 0; int do_silver_mueller = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 83f48394400..11646927863 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3370,3 +3370,127 @@ WarpX::AllocInitMultiFabFromModel ( } multifab_map[name_with_suffix] = mf.get(); } + +amrex::MultiFab* +WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const +{ + // This function does *not* check if the returned field pointer is != nullptr + + amrex::MultiFab* field_pointer = nullptr; + + switch(field_type) + { + case FieldType::Efield_aux : + field_pointer = Efield_aux[lev][direction].get(); + break; + case FieldType::Bfield_aux : + field_pointer = Bfield_aux[lev][direction].get(); + break; + case FieldType::Efield_fp : + field_pointer = Efield_fp[lev][direction].get(); + break; + case FieldType::Bfield_fp : + field_pointer = Bfield_fp[lev][direction].get(); + break; + case FieldType::current_fp : + field_pointer = current_fp[lev][direction].get(); + break; + case FieldType::current_fp_nodal : + field_pointer = current_fp_nodal[lev][direction].get(); + break; + case FieldType::rho_fp : + field_pointer = rho_fp[lev].get(); + break; + case FieldType::F_fp : + field_pointer = F_fp[lev].get(); + break; + case FieldType::G_fp : + field_pointer = G_fp[lev].get(); + break; + case FieldType::phi_fp : + field_pointer = phi_fp[lev].get(); + break; + case FieldType::vector_potential_fp : + field_pointer = vector_potential_fp_nodal[lev][direction].get(); + break; + case FieldType::Efield_cp : + field_pointer = Efield_cp[lev][direction].get(); + break; + case FieldType::Bfield_cp : + field_pointer = Bfield_cp[lev][direction].get(); + break; + case FieldType::current_cp : + field_pointer = current_cp[lev][direction].get(); + break; + case FieldType::rho_cp : + field_pointer = rho_cp[lev].get(); + break; + case FieldType::F_cp : + field_pointer = F_cp[lev].get(); + break; + case FieldType::G_cp : + field_pointer = G_cp[lev].get(); + break; + case FieldType::edge_lengths : + field_pointer = m_edge_lengths[lev][direction].get(); + break; + case FieldType::face_areas : + field_pointer = m_face_areas[lev][direction].get(); + break; + case FieldType::Efield_avg_fp : + field_pointer = Efield_avg_fp[lev][direction].get(); + break; + case FieldType::Bfield_avg_fp : + field_pointer = Bfield_avg_fp[lev][direction].get(); + break; + case FieldType::Efield_avg_cp : + field_pointer = Efield_avg_cp[lev][direction].get(); + break; + case FieldType::Bfield_avg_cp : + field_pointer = Bfield_avg_cp[lev][direction].get(); + break; + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + break; + } + + return field_pointer; +} + +bool +WarpX::isFieldInitialized (const FieldType field_type, const int lev, const int direction) const +{ + const bool is_field_init = (getFieldPointerUnchecked(field_type, lev, direction) != nullptr); + return is_field_init; +} + +amrex::MultiFab* +WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const +{ + auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_pointer != nullptr, "Requested field is not initialized!"); + return field_pointer; +} + +std::array +WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (field_type == FieldType::Efield_aux) || (field_type == FieldType::Bfield_aux) || + (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || + (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || + (field_type == FieldType::Efield_cp) || (field_type == FieldType::Bfield_cp) || + (field_type == FieldType::current_cp), "Requested field type is not a vector."); + + return std::array{ + getFieldPointer(field_type, lev, 0), + getFieldPointer(field_type, lev, 1), + getFieldPointer(field_type, lev, 2)}; +} + +const amrex::MultiFab& +WarpX::getField(FieldType field_type, const int lev, const int direction) const +{ + return *getFieldPointer(field_type, lev, direction); +} diff --git a/Source/ablastr/parallelization/MPIInitHelpers.H b/Source/ablastr/parallelization/MPIInitHelpers.H index 6b9bdfa4737..660e5e56d06 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.H +++ b/Source/ablastr/parallelization/MPIInitHelpers.H @@ -15,7 +15,7 @@ namespace ablastr::parallelization * * @return the MPI_THREAD_* level required for MPI_Init_thread */ - int + constexpr int mpi_thread_required (); /** Initialize MPI diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp index 633c004c93a..513035c483c 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.cpp +++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp @@ -29,20 +29,20 @@ namespace ablastr::parallelization { - int + constexpr int mpi_thread_required () { - int thread_required = -1; #ifdef AMREX_USE_MPI - thread_required = MPI_THREAD_SINGLE; // equiv. to MPI_Init -# ifdef AMREX_USE_OMP - thread_required = MPI_THREAD_FUNNELED; -# endif # ifdef AMREX_MPI_THREAD_MULTIPLE // i.e. for async_io - thread_required = MPI_THREAD_MULTIPLE; + return MPI_THREAD_MULTIPLE; +# elif AMREX_USE_OMP + return MPI_THREAD_FUNNELED; +# else + return MPI_THREAD_SINGLE; // equiv. to MPI_Init # endif +#else + return -1; #endif - return thread_required; } std::pair< int, int > @@ -58,7 +58,7 @@ namespace ablastr::parallelization } #endif - const int thread_required = mpi_thread_required(); + constexpr int thread_required = mpi_thread_required(); #ifdef AMREX_USE_MPI int thread_provided = -1; MPI_Init_thread(&argc, &argv, thread_required, &thread_provided); @@ -82,7 +82,7 @@ namespace ablastr::parallelization check_mpi_thread_level () { #ifdef AMREX_USE_MPI - const int thread_required = mpi_thread_required(); + constexpr int thread_required = mpi_thread_required(); int thread_provided = -1; MPI_Query_thread(&thread_provided); auto mtn = amrex::ParallelDescriptor::mpi_level_to_string; From 63807df9f1c63de6c71ee2ac0944a6cb17ce2262 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 1 Apr 2024 16:03:28 -0700 Subject: [PATCH 041/121] Doc: Perlmutter Update (#4809) - use a TMPFS build directory instead of HOME: less quota strain, faster - update optimas install: from PyPI, install torch on CPU, too - rename venvs: ensure -cpu or -gpu is seen on the prompt when activated --- .../install_cpu_dependencies.sh | 44 ++++++++++------- .../install_gpu_dependencies.sh | 47 ++++++++++--------- .../perlmutter_cpu_warpx.profile.example | 4 +- .../perlmutter_gpu_warpx.profile.example | 4 +- 4 files changed, 56 insertions(+), 43 deletions(-) diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index dbe59438a16..f65e43891d0 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -44,6 +44,9 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -55,9 +58,9 @@ else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi rm -rf $HOME/src/c-blosc-pm-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-pm-cpu-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pm-cpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -70,9 +73,9 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-cpu-build --target install -j 16 -rm -rf $HOME/src/adios2-pm-cpu-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j 16 +rm -rf ${build_dir}/adios2-pm-cpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -86,9 +89,9 @@ else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-cpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-pm-cpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pm-cpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -102,9 +105,9 @@ else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-cpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-pm-cpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pm-cpu-build # Python ###################################################################### @@ -112,9 +115,9 @@ rm -rf $HOME/src/lapackpp-pm-cpu-build python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv ${SW_DIR}/venvs/warpx -source ${SW_DIR}/venvs/warpx/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-cpu +python3 -m venv ${SW_DIR}/venvs/warpx-cpu +source ${SW_DIR}/venvs/warpx-cpu/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,7 +131,12 @@ MPICC="cc -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-b python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard +# install or update WarpX dependencies python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 3fe76359953..9ac5800c6ce 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -44,6 +44,9 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -55,9 +58,9 @@ else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi rm -rf $HOME/src/c-blosc-pm-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-pm-gpu-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pm-gpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -70,9 +73,9 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-gpu-build --target install -j 16 -rm -rf $HOME/src/adios2-pm-gpu-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j 16 +rm -rf ${build_dir}/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -86,9 +89,9 @@ else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-pm-gpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -102,9 +105,9 @@ else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-pm-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pm-gpu-build # Python ###################################################################### @@ -112,9 +115,9 @@ rm -rf $HOME/src/lapackpp-pm-gpu-build python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv ${SW_DIR}/venvs/warpx -source ${SW_DIR}/venvs/warpx/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-gpu +python3 -m venv ${SW_DIR}/venvs/warpx-gpu +source ${SW_DIR}/venvs/warpx-gpu/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,11 +131,13 @@ MPICC="cc -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4p python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard +# install or update WarpX dependencies python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -python3 -m pip install cupy-cuda12x # CUDA 12 compatible wheel -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) python3 -m pip install --upgrade torch # CUDA 12 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 67ae36c9f1a..1b0ac3182d5 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -33,9 +33,9 @@ export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/ # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx" ] +if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu" ] then - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx/bin/activate + source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate fi # an alias to request an interactive batch node for one hour diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 7bf5dd13116..759df0b923a 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -37,9 +37,9 @@ export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/ # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx" ] +if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu" ] then - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx/bin/activate + source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate fi # an alias to request an interactive batch node for one hour From 5a8fdd8d3ebfbaf56960716ee4773571c5ffe059 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Tue, 2 Apr 2024 06:22:08 -0700 Subject: [PATCH 042/121] Printing out the time dependencies of the charge and current densities with multi-J PSATD. (#4815) --- Source/Initialization/WarpXInitData.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5309b878f08..59459f7ad63 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -300,6 +300,18 @@ WarpX::PrintMainPICparameters () amrex::Print() << " | - multi-J deposition is ON \n"; amrex::Print() << " | - do_multi_J_n_depositions = " << WarpX::do_multi_J_n_depositions << "\n"; + if (J_in_time == JInTime::Linear){ + amrex::Print() << " | - J_in_time = linear \n"; + } + if (J_in_time == JInTime::Constant){ + amrex::Print() << " | - J_in_time = constant \n"; + } + if (rho_in_time == RhoInTime::Linear){ + amrex::Print() << " | - rho_in_time = linear \n"; + } + if (rho_in_time == RhoInTime::Constant){ + amrex::Print() << " | - rho_in_time = constant \n"; + } } if (fft_do_time_averaging == 1){ amrex::Print()<<" | - time-averaged is ON \n"; From 5e07c8047fcafd7fd945bfaa1fa78223d2738c10 Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Tue, 2 Apr 2024 16:18:07 -0700 Subject: [PATCH 043/121] updated docs about focal distance (#4762) * updated docs about focal distance --- Docs/source/refs.bib | 9 +++++++++ Docs/source/usage/parameters.rst | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 6c7665dac84..1746e2285cd 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -393,3 +393,12 @@ @article{GrismayerNJP2021 volume = {23}, year = {2021} } + +@book{Wiedemann2015, +author = {Wiedemann, H.}, +isbn = {978-3-319-18317-6}, +publisher = {Springer Cham}, +title = {{Particle Accelerator Physics}}, +doi = {https://doi.org/10.1007/978-3-319-18317-6}, +year = {2015} +} diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index fa1d5e99670..1eaf22f0183 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -810,7 +810,20 @@ Particle initialization * ``.focal_distance`` (optional, distance between the beam centroid and the position of the focal plane of the beam, along the direction of the beam mean velocity; space charge is ignored in the initialization of the particles) - If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the size of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the sizes of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + + Usually, in accelerator physics the operative quantities are the normalized emittances :math:`\epsilon_{x,y}` and beta functions :math:`\beta_{x,y}`. + We assume that the beam travels along :math:`z` and we mark the quantities evaluated at the focal plane with a :math:`*`. + Therefore, the normalized transverse emittances and beta functions are related to the focal distance :math:`f = z - z^*`, the beam sizes :math:`\sigma_{x,y}` (which in the code are ``x_rms``, ``y_rms``), the beam relativistic Lorentz factor :math:`\gamma`, and the normalized momentum spread :math:`\Delta u_{x,y}` according to the equations below (:cite:t:`param-Wiedemann2015`). + + .. math:: + + \Delta u_{x,y} &= \frac{\epsilon^*_{x,y}}{\sigma^*_{x,y}}, + + \sigma*_{x, y} &= \sqrt{ \frac{ \epsilon^*_{x,y} \beta^*_{x,y} }{\gamma}}, + + \sigma_{x,y}(z) &= \sigma^*_{x,y} \sqrt{1 + \left( \frac{z - z^*}{\beta^*_{x,y}} \right)^2} + * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: From 30ed0fbac4c0c4cf47f10e2a255c52c50166c79f Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Wed, 3 Apr 2024 16:27:47 -0700 Subject: [PATCH 044/121] Implement Poisson solver based on integrated Green functions (#4648) * Allocate 2x rho * Copy rho * Allocate G * Implemented Green function * Filled values of the integrated Green function * Call computePhiIGF * Added test script and correct periodicity * Do not call the Poisson solver if the charge is 0 * Correct compilation errors * Perform copy back to phi * Correct a few bugs * Improve comparison with theory * Simplify implementation * Add relativistic factor * Use new distribution mapping * Use ParallelCopy for rho and phi * Use correct DistributionMapping * Cleaner implementation with nx,ny,nz * More compact syntax for BoxArrays * Avoid floating-point error * Fixes to avoid issues with photon containers and empty containers * Add more timers * Fix ParallelCopy usage * Use Taylor expansion for pathological terms * Use more accurate expression * Update Source/ablastr/fields/PoissonSolver.H * Correctly fill ghost cells of phi * Update Green function formula * start automated test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add warpx.poisson_solver to test * moved computePhiIGF in separate file * updated test analysis * add user interface and checks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed BC assert * add ifdefs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed unused import * fix bracket * fix checks * check 3D and PSATD * fix and add asserts * fix rho mult by eps0 * fix typo * update test * reset benchmark from azure * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix clang-tidy tantrums * add literals * other checks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * forgot WarpX:: * forgot semicolons :| * docs and comments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * forgot semicolon again * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/math/fft/WrapCuFFT.cpp Co-authored-by: Axel Huebl * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl * false for device syncs * remove WarpX:: * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add maybe unused * fixed solver check * update test inputs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/WarpX.cpp Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/Utils/WarpXUtil.cpp Co-authored-by: Axel Huebl * Update Source/Initialization/WarpXInitData.cpp Co-authored-by: Axel Huebl * renamed solvers * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * avoid rescaling rho twice * fix doxy string * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Simplify is_solver_multigrid * Update Docs/source/usage/parameters.rst Co-authored-by: Axel Huebl * Update Docs/source/usage/parameters.rst Co-authored-by: Axel Huebl * Update Examples/Tests/openbc_poisson_solver/inputs_3d Co-authored-by: Axel Huebl * Update Docs/source/refs.bib Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * adds cpp in ablastr/fields * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add Missing Files * `BaseFab`: Fix Complex Type All complex types are equal, but some complex types are more equal than others. -- George Orwell * Cleaning: Includes * Cleaning: Const Refs and Doxygen * Update Source/Utils/WarpXAlgorithmSelection.cpp * Update Source/FieldSolver/ElectrostaticSolver.cpp * Fix .bib File Syntax * Update Regression/Checksum/benchmarks_json/openbc_poisson_solver.json --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Tools Co-authored-by: Axel Huebl --- Docs/source/refs.bib | 40 +++- Docs/source/usage/parameters.rst | 36 +++- .../Tests/openbc_poisson_solver/analysis.py | 65 ++++++ .../Tests/openbc_poisson_solver/inputs_3d | 46 ++++ .../openbc_poisson_solver.json | 20 ++ Regression/WarpX-tests.ini | 18 ++ Source/FieldSolver/ElectrostaticSolver.cpp | 43 +++- Source/Initialization/WarpXInitData.cpp | 13 +- Source/Utils/WarpXAlgorithmSelection.H | 12 +- Source/Utils/WarpXAlgorithmSelection.cpp | 9 + Source/Utils/WarpXUtil.cpp | 9 + Source/WarpX.H | 1 + Source/WarpX.cpp | 22 ++ Source/ablastr/CMakeLists.txt | 2 +- Source/ablastr/Make.package | 1 + Source/ablastr/fields/CMakeLists.txt | 9 + .../fields/IntegratedGreenFunctionSolver.H | 66 ++++++ .../fields/IntegratedGreenFunctionSolver.cpp | 196 ++++++++++++++++++ Source/ablastr/fields/Make.package | 7 + Source/ablastr/fields/PoissonSolver.H | 33 ++- Source/ablastr/math/fft/WrapCuFFT.cpp | 4 + 21 files changed, 625 insertions(+), 27 deletions(-) create mode 100755 Examples/Tests/openbc_poisson_solver/analysis.py create mode 100644 Examples/Tests/openbc_poisson_solver/inputs_3d create mode 100644 Regression/Checksum/benchmarks_json/openbc_poisson_solver.json create mode 100644 Source/ablastr/fields/CMakeLists.txt create mode 100644 Source/ablastr/fields/IntegratedGreenFunctionSolver.H create mode 100644 Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp create mode 100644 Source/ablastr/fields/Make.package diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 1746e2285cd..2de27908c5d 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -394,11 +394,39 @@ @article{GrismayerNJP2021 year = {2021} } +@article{QiangPhysRevSTAB2006, + title = {Three-dimensional quasistatic model for high brightness beam dynamics simulation}, + author = {Qiang, Ji and Lidia, Steve and Ryne, Robert D. and Limborg-Deprey, Cecile}, + journal = {Phys. Rev. ST Accel. Beams}, + volume = {9}, + issue = {4}, + pages = {044204}, + numpages = {10}, + year = {2006}, + month = {Apr}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevSTAB.9.044204} +} + +@article{QiangPhysRevSTAB2006err, + title = {Erratum: Three-dimensional quasistatic model for high brightness beam dynamics simulation [Phys. Rev. ST Accel. Beams 9, 044204 (2006)]}, + author = {Qiang, Ji and Lidia, Steve and Ryne, Robert D. and Limborg-Deprey, Cecile}, + journal = {Phys. Rev. ST Accel. Beams}, + volume = {10}, + issue = {12}, + pages = {129901}, + numpages = {2}, + year = {2007}, + month = {Dec}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevSTAB.10.129901} +} + @book{Wiedemann2015, -author = {Wiedemann, H.}, -isbn = {978-3-319-18317-6}, -publisher = {Springer Cham}, -title = {{Particle Accelerator Physics}}, -doi = {https://doi.org/10.1007/978-3-319-18317-6}, -year = {2015} + author = {Wiedemann, H.}, + isbn = {978-3-319-18317-6}, + publisher = {Springer Cham}, + title = {{Particle Accelerator Physics}}, + doi = {10.1007/978-3-319-18317-6}, + year = {2015} } diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 1eaf22f0183..4ea9a3dc65d 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -157,16 +157,28 @@ Overall simulation parameters where :math:`\boldsymbol{\beta}` is the average (normalized) velocity of the considered species (which can be relativistic). See, e.g., :cite:t:`param-Vaypop2008` for more information. - See the `AMReX documentation `_ - for details of the MLMG solver (the default solver used with electrostatic - simulations). The default behavior of the code is to check whether there is - non-zero charge density in the system and if so force the MLMG solver to - use the solution max norm when checking convergence. If there is no charge - density, the MLMG solver will switch to using the initial guess max norm - error when evaluating convergence and an absolute error tolerance of - :math:`10^{-6}` :math:`\mathrm{V/m}^2` will be used (unless a different - non-zero value is specified by the user via - ``warpx.self_fields_absolute_tolerance``). +* ``warpx.poisson_solver`` (`string`) optional (default `multigrid`) + + * ``multigrid``: Poisson's equation is solved using an iterative multigrid (MLMG) solver. + See the `AMReX documentation `__ + for details of the MLMG solver (the default solver used with electrostatic + simulations). The default behavior of the code is to check whether there is + non-zero charge density in the system and if so force the MLMG solver to + use the solution max norm when checking convergence. If there is no charge + density, the MLMG solver will switch to using the initial guess max norm + error when evaluating convergence and an absolute error tolerance of + :math:`10^{-6}` :math:`\mathrm{V/m}^2` will be used (unless a different + non-zero value is specified by the user via + ``warpx.self_fields_absolute_tolerance``). + + * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). + See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. + It only works in 3D and it requires the compilation flag ``-DWarpX_PSATD=ON``. + If mesh refinement is enabled, this solver only works on the coarsest level. + On the refined patches, the Poisson equation is solved with the multigrid solver. + In electrostatic mode, this solver requires open field boundary conditions (``boundary.field_lo,hi = open``). + In electromagnetic mode, this solver can be used to initialize the species' self fields + (``.initialize_self_fields=1``) provided that the field BCs are PML (``boundary.field_lo,hi = PML``). * ``warpx.self_fields_required_precision`` (`float`, default: 1.e-11) The relative precision with which the electrostatic space-charge fields should @@ -398,7 +410,9 @@ Domain Boundary Conditions * ``none``: No boundary condition is applied to the fields with the electromagnetic solver. This option must be used for the RZ-solver at `r=0`. - * ``neumann``: For the electrostatic solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. + * ``neumann``: For the electrostatic multigrid solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. + + * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method. * ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers diff --git a/Examples/Tests/openbc_poisson_solver/analysis.py b/Examples/Tests/openbc_poisson_solver/analysis.py new file mode 100755 index 00000000000..dc7b5dca8bd --- /dev/null +++ b/Examples/Tests/openbc_poisson_solver/analysis.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import epsilon_0, pi +from scipy.special import erf + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +sigmaz = 300e-6 +sigmax = 516e-9 +sigmay = 7.7e-9 +Q = -3.2e-9 + +def w(z): + return np.exp(-z**2) * ( 1 + erf(1.j*z) ) + +def evaluate_E(x, y, z): + ''' Basseti-Erskine formula https://cds.cern.ch/record/122227/files/198005132.pdf ''' + den = np.sqrt(2*(sigmax**2-sigmay**2)) + arg1 = (x+1j*y)/den + term1 = w(arg1) + arg2 = (x*sigmay/sigmax + 1j*y*sigmax/sigmay)/den + term2 = -np.exp(-x**2/(2*sigmax**2)-y**2/(2*sigmay**2))*w(arg2) + factor = Q/(2.*np.sqrt(2.)*pi*epsilon_0*sigmaz*den)*np.exp(-z**2/(2*sigmaz**2)) + E_complex = factor*(term1 + term2) + return E_complex.imag, E_complex.real + + +fn = sys.argv[1] + +path=os.path.join('diags', 'diag2') +ts = OpenPMDTimeSeries(path) + +Ex, info = ts.get_field(field='E', coord='x', iteration=0, plot=False) +Ey, info = ts.get_field(field='E', coord='y', iteration=0, plot=False) + +grid_x = info.x[1:-1] +grid_y = info.y[1:-1] +grid_z = info.z[1:-1] + +hnx = int(0.5*len(grid_x)) +hny = int(0.5*len(grid_y)) + +# Compare theoretical and WarpX Ex, Ey fields for every z +for k, z in enumerate(grid_z,start=1): + Ex_warpx = Ex[k,hny,1:-1] + Ey_warpx = Ey[k,1:-1,hnx] + + Ex_theory = evaluate_E(grid_x, 0., z)[0] + Ey_theory = evaluate_E(0., grid_y, z)[1] + + assert(np.allclose(Ex_warpx, Ex_theory, rtol=0.032, atol=0)) + assert(np.allclose(Ey_warpx, Ey_theory, rtol=0.029, atol=0)) + + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-2) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/inputs_3d new file mode 100644 index 00000000000..fd4299f360b --- /dev/null +++ b/Examples/Tests/openbc_poisson_solver/inputs_3d @@ -0,0 +1,46 @@ +my_constants.sigmaz = 300e-6 +my_constants.sigmax = 516e-9 +my_constants.sigmay = 7.7e-9 +my_constants.Q = 3.2e-9 + +max_step = 1 +amr.n_cell = 128 128 128 +amr.max_level = 0 +amr.max_grid_size = 128 +geometry.dims = 3 +geometry.prob_lo = -5*sigmax -5*sigmay -5*sigmaz +geometry.prob_hi = +5*sigmax +5*sigmay +5*sigmaz +boundary.field_lo = open open open +boundary.field_hi = open open open +warpx.const_dt = 1e-14 +warpx.do_electrostatic = relativistic +warpx.poisson_solver = fft +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + + +particles.species_names = electron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 2 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "Q/(sqrt(2*pi)**3 * sigmax*sigmay*sigmaz * q_e) * exp( -x*x/(2*sigmax*sigmax) -y*y/(2*sigmay*sigmay) - z*z/(2*sigmaz*sigmaz) )" +electron.momentum_distribution_type = "constant" +electron.ux = 0.0 +electron.uy = 0.0 +electron.uz = 25000 +electron.initialize_self_fields = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho +diag1.format = plotfile + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Ex Ey +diag2.format = openpmd diff --git a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json new file mode 100644 index 00000000000..50b402b4bab --- /dev/null +++ b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 67876594.52390265, + "By": 112770930.05280717, + "Bz": 1.1060167389533003e-13, + "Ex": 3.380787637240387e+16, + "Ey": 2.034889235345363e+16, + "Ez": 2601470.9380206033, + "rho": 5630115646680.221 + }, + "electron": { + "particle_momentum_x": 6.378073505535043e-19, + "particle_momentum_y": 3.838951101276526e-19, + "particle_momentum_z": 1.1454327682972433e-10, + "particle_position_x": 21.64260864000591, + "particle_position_y": 0.3229614080035443, + "particle_position_z": 12582.912000000004, + "particle_weight": 19972794743.418434 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 897fcde3cf9..e9c53bb37fd 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4737,3 +4737,21 @@ compileTest = 0 doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py + +[openbc_poisson_solver] +buildDir = . +inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +doComparison = 0 +analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 755c9d6e2a0..8f022c308af 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -146,6 +146,10 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) { WARPX_PROFILE("WarpX::AddSpaceChargeField"); + if (pc.getCharge() == 0) { + return; + } + // Store the boundary conditions for the field solver if they haven't been // stored yet if (!m_poisson_boundary_handler.bcs_set) { @@ -249,7 +253,7 @@ WarpX::AddSpaceChargeFieldLabFrame () // Use the tridiag solver with 1D computePhiTriDiagonal(rho_fp, phi_fp); #else - // Use the AMREX MLMG solver otherwise + // Use the AMREX MLMG or the FFT (IGF) solver otherwise computePhi(rho_fp, phi_fp, beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity); @@ -352,6 +356,9 @@ WarpX::computePhi (const amrex::Vector >& rho, const std::optional > eb_farray_box_factory; #endif + bool const is_solver_multigrid = + WarpX::poisson_solver_id != PoissonSolverAlgo::IntegratedGreenFunction; + ablastr::fields::computePhi( sorted_rho, sorted_phi, @@ -364,6 +371,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->dmap, this->grids, this->m_poisson_boundary_handler, + is_solver_multigrid, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, @@ -1001,6 +1009,7 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo amrex::ignore_unused(geom); #endif for (int idim=dim_start; idim electrostatic_solver_algo_to_int = { {"default", ElectrostaticSolverAlgo::None } }; +const std::map poisson_solver_algo_to_int = { + {"multigrid", PoissonSolverAlgo::Multigrid}, + {"fft", PoissonSolverAlgo::IntegratedGreenFunction}, + {"default", PoissonSolverAlgo::Multigrid } +}; + const std::map particle_pusher_algo_to_int = { {"boris", ParticlePusherAlgo::Boris }, {"vay", ParticlePusherAlgo::Vay }, @@ -125,6 +131,7 @@ const std::map FieldBCType_algo_to_int = { {"damped", FieldBoundaryType::Damped}, {"absorbing_silver_mueller", FieldBoundaryType::Absorbing_SilverMueller}, {"neumann", FieldBoundaryType::Neumann}, + {"open", FieldBoundaryType::Open}, {"none", FieldBoundaryType::None}, {"default", FieldBoundaryType::PML} }; @@ -163,6 +170,8 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ algo_to_int = grid_to_int; } else if (0 == std::strcmp(pp_search_key, "do_electrostatic")) { algo_to_int = electrostatic_solver_algo_to_int; + } else if (0 == std::strcmp(pp_search_key, "poisson_solver")) { + algo_to_int = poisson_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "particle_pusher")) { algo_to_int = particle_pusher_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "current_deposition")) { diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index e08a6304f09..d2691685c53 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -404,6 +404,7 @@ void ReadBCParams () const ParmParse pp_warpx("warpx"); const ParmParse pp_algo("algo"); const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); + const int poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); if (pp_geometry.queryarr("is_periodic", geom_periodicity)) { @@ -469,6 +470,14 @@ void ReadBCParams () ), "PEC boundary not implemented for PSATD, yet!" ); + + if(WarpX::field_boundary_lo[idim] == FieldBoundaryType::Open && + WarpX::field_boundary_hi[idim] == FieldBoundaryType::Open){ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction, + "Field open boundary conditions are only implemented for the FFT-based Poisson solver" + ); + } } // Appending periodicity information to input so that it can be used by amrex diff --git a/Source/WarpX.H b/Source/WarpX.H index 3be95f47865..cdd2447ed5e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -929,6 +929,7 @@ public: static const amrex::iMultiFab* GatherBufferMasks (int lev); static int electrostatic_solver_id; + static int poisson_solver_id; // Parameters for lab frame electrostatic static amrex::Real self_fields_required_precision; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 11646927863..f89a61e524b 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -190,6 +190,7 @@ amrex::IntVect WarpX::sort_idx_type(AMREX_D_DECL(0,0,0)); bool WarpX::do_dynamic_scheduling = true; int WarpX::electrostatic_solver_id; +int WarpX::poisson_solver_id; Real WarpX::self_fields_required_precision = 1.e-11_rt; Real WarpX::self_fields_absolute_tolerance = 0.0_rt; int WarpX::self_fields_max_iters = 200; @@ -763,6 +764,27 @@ WarpX::ReadParameters () pp_warpx, "self_fields_max_iters", self_fields_max_iters); pp_warpx.query("self_fields_verbosity", self_fields_verbosity); } + + poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); +#ifndef WARPX_DIM_3D + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, + "The FFT Poisson solver only works in 3D."); +#endif +#ifndef WARPX_USE_PSATD + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, + "To use the FFT Poisson solver, compile with WARPX_USE_PSATD=ON."); +#endif + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + ( + electrostatic_solver_id!=ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic || + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction + ), + "The FFT Poisson solver is not implemented in labframe-electromagnetostatic mode yet." + ); + // Parse the input file for domain boundary potentials const ParmParse pp_boundary("boundary"); bool potential_specified = false; diff --git a/Source/ablastr/CMakeLists.txt b/Source/ablastr/CMakeLists.txt index a41fa4ae3a0..03b876fa1ab 100644 --- a/Source/ablastr/CMakeLists.txt +++ b/Source/ablastr/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(coarsen) add_subdirectory(math) -#add_subdirectory(fields) +add_subdirectory(fields) add_subdirectory(parallelization) #add_subdirectory(particles) #add_subdirectory(profiler) diff --git a/Source/ablastr/Make.package b/Source/ablastr/Make.package index 3426020e45d..b9ff3c72560 100644 --- a/Source/ablastr/Make.package +++ b/Source/ablastr/Make.package @@ -2,6 +2,7 @@ include $(WARPX_HOME)/Source/ablastr/coarsen/Make.package include $(WARPX_HOME)/Source/ablastr/math/Make.package +include $(WARPX_HOME)/Source/ablastr/fields/Make.package include $(WARPX_HOME)/Source/ablastr/parallelization/Make.package include $(WARPX_HOME)/Source/ablastr/particles/Make.package include $(WARPX_HOME)/Source/ablastr/utils/Make.package diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt new file mode 100644 index 00000000000..f7dfb6763ec --- /dev/null +++ b/Source/ablastr/fields/CMakeLists.txt @@ -0,0 +1,9 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + if(WarpX_PSATD AND D EQUAL 3) + target_sources(ablastr_${SD} + PRIVATE + IntegratedGreenFunctionSolver.cpp + ) + endif() +endforeach() diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H new file mode 100644 index 00000000000..97ffdb5ac36 --- /dev/null +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H @@ -0,0 +1,66 @@ +/* Copyright 2019-2024 Arianna Formenti, Remi Lehe + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef ABLASTR_IGF_SOLVER_H +#define ABLASTR_IGF_SOLVER_H + +#include +#include +#include +#include +#include + + +#include +#include + + +namespace ablastr::fields +{ + + /** @brief Implements equation 2 in https://doi.org/10.1103/PhysRevSTAB.10.129901 + * with some modification to symmetrize the function. + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * @param[in] z z-coordinate of given location + * + * @return the integrated Green function G + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + IntegratedPotential (amrex::Real x, amrex::Real y, amrex::Real z) + { + using namespace amrex::literals; + + amrex::Real const r = std::sqrt( x*x + y*y + z*z ); + amrex::Real const G = + - 0.5_rt * z*z * std::atan( x*y/(z*r) ) + - 0.5_rt * y*y * std::atan( x*z/(y*r) ) + - 0.5_rt * x*x * std::atan( y*z/(x*r) ) + + y*z*std::asinh( x/std::sqrt(y*y + z*z) ) + + x*z*std::asinh( y/std::sqrt(x*x + z*z) ) + + x*y*std::asinh( z/std::sqrt(x*x + y*y) ); + return G; + } + + /** @brief Compute the electrostatic potential using the Integrated Green Function method + * as in http://dx.doi.org/10.1103/PhysRevSTAB.9.044204 + * + * @param[in] rho the charge density amrex::MultiFab + * @param[out] phi the electrostatic potential amrex::MultiFab + * @param[in] cell_size an arreay of 3 reals dx dy dz + * @param[in] ba amrex::BoxArray with the grid of a given level + */ + void + computePhiIGF (amrex::MultiFab const & rho, + amrex::MultiFab & phi, + std::array const & cell_size, + amrex::BoxArray const & ba); + +} // namespace ablastr::fields + +#endif // ABLASTR_IGF_SOLVER_H diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp new file mode 100644 index 00000000000..0767ecfb2f3 --- /dev/null +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -0,0 +1,196 @@ +/* Copyright 2019-2024 Arianna Formenti, Remi Lehe + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ +#include "IntegratedGreenFunctionSolver.H" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace ablastr::fields { + +void +computePhiIGF ( amrex::MultiFab const & rho, + amrex::MultiFab & phi, + std::array const & cell_size, + amrex::BoxArray const & ba ) +{ + using namespace amrex::literals; + + // Define box that encompasses the full domain + amrex::Box domain = ba.minimalBox(); + domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal + domain.grow( phi.nGrowVect() ); // include guard cells + + int const nx = domain.length(0); + int const ny = domain.length(1); + int const nz = domain.length(2); + + // Allocate 2x wider arrays for the convolution of rho with the Green function + // This also defines the box arrays for the global FFT: contains only one box; + amrex::Box const realspace_box = amrex::Box( + {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, + {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, + amrex::IntVect::TheNodeVector() ); + amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); + amrex::Box const spectralspace_box = amrex::Box( + {0,0,0}, + {nx, 2*ny-1, 2*nz-1}, + amrex::IntVect::TheNodeVector() ); + amrex::BoxArray const spectralspace_ba = amrex::BoxArray( spectralspace_box ); + // Define a distribution mapping for the global FFT, with only one box + amrex::DistributionMapping dm_global_fft; + dm_global_fft.define( realspace_ba ); + // Allocate required arrays + amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); + tmp_rho.setVal(0); + amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); + tmp_G.setVal(0); + // Allocate corresponding arrays in Fourier space + using SpectralField = amrex::FabArray< amrex::BaseFab< amrex::GpuComplex< amrex::Real > > >; + SpectralField tmp_rho_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); + SpectralField tmp_G_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); + + // Copy from rho to tmp_rho + tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); + + // Compute the integrated Green function + { + BL_PROFILE("Initialize Green function"); + amrex::BoxArray const domain_ba = amrex::BoxArray( domain ); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(domain_ba, dm_global_fft,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + amrex::Box const bx = mfi.tilebox(); + + amrex::IntVect const lo = realspace_box.smallEnd(); + amrex::IntVect const hi = realspace_box.bigEnd(); + + // Fill values of the Green function + amrex::Real const dx = cell_size[0]; + amrex::Real const dy = cell_size[1]; + amrex::Real const dz = cell_size[2]; + amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); + amrex::ParallelFor( bx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + int const k0 = k - lo[2]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::Real const z = k0*dz; + + amrex::Real const G_value = 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( + IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + ); + + tmp_G_arr(i,j,k) = G_value; + // Fill the rest of the array by periodicity + if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} + if (j0>0) {tmp_G_arr(i , hi[1]+1-j0, k ) = G_value;} + if (k0>0) {tmp_G_arr(i , j , hi[2]+1-k0) = G_value;} + if ((i0>0)&&(j0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, k ) = G_value;} + if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} + if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} + if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} + } + ); + } + } + + // Perform forward FFTs + auto forward_plan_rho = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + auto forward_plan_G = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + // Loop over boxes perform FFTs + for ( amrex::MFIter mfi(realspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + + // Note: the size of the real-space box and spectral-space box + // differ when using real-to-complex FFT. When initializing + // the FFT plan, the valid dimensions are those of the real-space box. + const amrex::IntVect fft_size = realspace_ba[mfi].length(); + + // FFT of rho + forward_plan_rho[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_rho[mfi].dataPtr(), + reinterpret_cast(tmp_rho_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(forward_plan_rho[mfi]); + + // FFT of G + forward_plan_G[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[mfi].dataPtr(), + reinterpret_cast(tmp_G_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(forward_plan_G[mfi]); + + } + + // Multiply tmp_G_fft and tmp_rho_fft in spectral space + // Store the result in-place in Gtmp_G_fft, to save memory + amrex::Multiply( tmp_G_fft, tmp_rho_fft, 0, 0, 1, 0); + + // Perform inverse FFT + auto backward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + // Loop over boxes perform FFTs + for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + + // Note: the size of the real-space box and spectral-space box + // differ when using real-to-complex FFT. When initializing + // the FFT plan, the valid dimensions are those of the real-space box. + const amrex::IntVect fft_size = realspace_ba[mfi].length(); + + // Inverse FFT: is done in-place, in the array of G + backward_plan[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[mfi].dataPtr(), + reinterpret_cast( tmp_G_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(backward_plan[mfi]); + } + // Normalize, since (FFT + inverse FFT) results in a factor N + const amrex::Real normalization = 1._rt / realspace_box.numPts(); + tmp_G.mult( normalization ); + + // Copy from tmp_G to phi + phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect() ); + + // Loop to destroy FFT plans + for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + ablastr::math::anyfft::DestroyPlan(forward_plan_G[mfi]); + ablastr::math::anyfft::DestroyPlan(forward_plan_rho[mfi]); + ablastr::math::anyfft::DestroyPlan(backward_plan[mfi]); + } +} +} // namespace ablastr::fields diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package new file mode 100644 index 00000000000..4fb7bd88e5f --- /dev/null +++ b/Source/ablastr/fields/Make.package @@ -0,0 +1,7 @@ +ifeq ($(USE_PSATD),TRUE) + ifeq ($(DIM),3) + CEXE_sources += IntegratedGreenFunctionSolver.cpp + endif +endif + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr/fields diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 77c06f11cf2..3489991671c 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -11,8 +11,12 @@ #include #include #include +#include #include +#if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) +#include +#endif #include #include @@ -77,6 +81,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler + * \param[in] is_solver_multigrid boolean to select the Poisson solver: 1 for Multigrid, 0 for FFT * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -100,6 +105,7 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& dmap, amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, + [[maybe_unused]] bool is_solver_multigrid, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, @@ -109,6 +115,8 @@ computePhi (amrex::Vector const & rho, { using namespace amrex::literals; + ABLASTR_PROFILE("computePhi", false); + if (!rel_ref_ratio.has_value()) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, "rel_ref_ratio must be set if mesh-refinement is used"); @@ -117,11 +125,9 @@ computePhi (amrex::Vector const & rho, auto const finest_level = static_cast(rho.size() - 1); - // scale rho appropriately; also determine if rho is zero everywhere + // determine if rho is zero everywhere amrex::Real max_norm_b = 0.0; for (int lev=0; lev<=finest_level; lev++) { - using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! max_norm_b = amrex::max(max_norm_b, rho[lev]->norm0()); } amrex::ParallelDescriptor::ReduceRealMax(max_norm_b); @@ -153,6 +159,27 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif +#if (defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D)) + // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected + if(!is_solver_multigrid && lev==0){ + amrex::Array const dx_igf + {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), + geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), + geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; + if ( max_norm_b == 0 ) { + phi[lev]->setVal(0); + } else { + computePhiIGF( *rho[lev], *phi[lev], dx_igf, grids[lev] ); + } + continue; + } +#endif + + // Use the Multigrid (MLMG) solver if selected or on refined patches + // but first scale rho appropriately + using namespace ablastr::constant::SI; + rho[lev]->mult(-1._rt/PhysConst::ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + #if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) // Determine whether to use semi-coarsening amrex::Array dx_scaled diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index 8ecea2a2486..b6b1706f807 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -8,6 +8,7 @@ #include "AnyFFT.H" #include "ablastr/utils/TextMsg.H" +#include "ablastr/profiler/ProfilerWrapper.H" namespace ablastr::math::anyfft { @@ -30,6 +31,7 @@ namespace ablastr::math::anyfft Complex * const complex_array, const direction dir, const int dim) { FFTplan fft_plan; + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan", false); // Initialize fft_plan.m_plan with the vendor fft plan. cufftResult result; @@ -69,10 +71,12 @@ namespace ablastr::math::anyfft void DestroyPlan(FFTplan& fft_plan) { + ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan", false); cufftDestroy( fft_plan.m_plan ); } void Execute(FFTplan& fft_plan){ + ABLASTR_PROFILE("ablastr::math::anyfft::Execute", false); // make sure that this is done on the same GPU stream as the above copy cudaStream_t stream = amrex::Gpu::Device::cudaStream(); cufftSetStream ( fft_plan.m_plan, stream); From f013b1f8601bfd883ae142f5fdf8054eecbaefb1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 3 Apr 2024 19:34:52 -0700 Subject: [PATCH 045/121] Fix: ABLASTR FFT Poisson Solver (#4821) Small compile issues seen in ImpactX. --- Source/ablastr/fields/PoissonSolver.H | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 3489991671c..da0078f8b5a 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -13,6 +13,7 @@ #include #include #include +#include #if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) #include @@ -178,7 +179,7 @@ computePhi (amrex::Vector const & rho, // Use the Multigrid (MLMG) solver if selected or on refined patches // but first scale rho appropriately using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/PhysConst::ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! #if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) // Determine whether to use semi-coarsening From febd6da36c6900aff8cd3308199dde60f3f86b0d Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:29:26 -0700 Subject: [PATCH 046/121] repetitive member var in derived class (#4827) --- Source/Diagnostics/BTDiagnostics.H | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 679dd6b8fa4..d5dd67226b7 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -176,9 +176,6 @@ private: /** Vector of lab-frame time corresponding to each snapshot */ amrex::Vector m_t_lab; - /** Vector of user-defined physical region for diagnostics in lab-frame - * for each back-transformed snapshot */ - amrex::Vector m_snapshot_domain_lab; /** Vector of physical region corresponding to the buffer that spans a part * of the full back-transformed snapshot */ amrex::Vector m_buffer_domain_lab; From 87e07677d927a9182940add1d8450c9958c9417e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 5 Apr 2024 09:39:52 -0700 Subject: [PATCH 047/121] Release 24.04 (#4824) * AMReX: 24.04 * pyAMReX: 24.04 * WarpX: 24.04 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 259c0d8d351..bfcd0d3e44b 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - + cd ../amrex && git checkout --detach 24.04 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 55fae3db78d..92f235d91ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.03) +project(WarpX VERSION 24.04) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 10752c21bda..7299b865746 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.03' +version = u'24.04' # The full version, including alpha/beta/rc tags. -release = u'24.03' +release = u'24.04' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 08e616cc279..d1b8ddd27e5 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.03', + version = '24.04', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f0ce221df0c..65f3de6c43a 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 +branch = 24.04 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index e9c53bb37fd..b4db494516d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 +branch = 24.04 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index db230573853..aa7c4ead995 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.03 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.04 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "47347f785f5c26f1f9e65bef8e10f2d383406ef7" +set(WarpX_amrex_branch "24.04" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 8ea26792217..cc43b9e3a29 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.03 CONFIG REQUIRED) + find_package(pyAMReX 24.04 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "b41b83d1c0f37c66ad0543afa7edaca9e0c82565" +set(WarpX_pyamrex_branch "24.04" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index f74eb50af8c..400ccf1737c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - +cd amrex && git checkout --detach 24.04 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index b0f919d707a..70b333fc6b9 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.03', + version = '24.04', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From e7af8bfd463237119d5ec1fd38d359374f423516 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:41:09 -0700 Subject: [PATCH 048/121] Fix outdated element-wise comparison based assertions (#4829) * Fix outdated element-wise comparison based assertions * Fix `<` to `<=` Co-authored-by: Weiqun Zhang --------- Co-authored-by: Weiqun Zhang --- Source/Parallelization/WarpXComm.cpp | 12 ++++++------ Source/ablastr/coarsen/sample.cpp | 4 ++-- Source/ablastr/particles/DepositCharge.H | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 7d5b25b1560..9daf37af3f8 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -596,7 +596,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In for (int i = 0; i < 3; ++i) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= mf[i]->nGrowVect(), + ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; @@ -653,7 +653,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In for (int i = 0; i < 3; ++i) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= mf[i]->nGrowVect(), + ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; @@ -684,7 +684,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Efield_avg_fp[lev][0]->nGrowVect(), + ng.allLE(Efield_avg_fp[lev][0]->nGrowVect()), "Error: in FillBoundaryE_avg, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Efield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Efield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -705,7 +705,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Efield_avg_cp[lev][0]->nGrowVect(), + ng.allLE(Efield_avg_cp[lev][0]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Efield_avg_cp[lev][0], ng, WarpX::do_single_precision_comms, cperiod); ablastr::utils::communication::FillBoundary(*Efield_avg_cp[lev][1], ng, WarpX::do_single_precision_comms, cperiod); @@ -737,7 +737,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Bfield_fp[lev][0]->nGrowVect(), + ng.allLE(Bfield_fp[lev][0]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -757,7 +757,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Bfield_avg_cp[lev][0]->nGrowVect(), + ng.allLE(Bfield_avg_cp[lev][0]->nGrowVect()), "Error: in FillBoundaryB_avg, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_cp[lev][0], ng, WarpX::do_single_precision_comms, cperiod); ablastr::utils::communication::FillBoundary(*Bfield_avg_cp[lev][1], ng, WarpX::do_single_precision_comms, cperiod); diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index b5661037c7e..51dcfc7dd08 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -41,12 +41,12 @@ namespace ablastr::coarsen::sample const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) { + if ( crse_ratio.allGT(amrex::IntVect(1)) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); } - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect().allGE(stag_dst-stag_src+ngrowvect), "source fine MultiFab does not have enough guard cells for this interpolation" ); // Auxiliary integer arrays (always 3D) diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index 4d78e309f85..b2dedccd03e 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -71,7 +71,7 @@ deposit_charge (typename T_PC::ParIterType& pti, if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); } - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho.allLE(rho->nGrowVect()), "num_rho_deposition_guards are larger than allocated!"); // particle shape auto const[nox, noy, noz] = std::array{particle_shape, particle_shape, particle_shape}; From 4a37ffbe1a6e2737b2794477946255730bc48566 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:11:16 -0700 Subject: [PATCH 049/121] Fix out of bounds memory access in `HybridPICModel::GetCurrentExternal` (#4830) --- .../HybridPICModel/HybridPICModel.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 228201886eb..6a688ce38b3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -97,12 +97,13 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri // the external current density multifab is made nodal to avoid needing to interpolate // to a nodal grid as has to be done for the ion and total current density multifabs + // this also allows the external current multifab to not have any ghost cells WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[x]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[x]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[y]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[y]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[z]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[z]", 0.0_rt); #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( From ef6d0853b0115dbbaf95b74d87c3367a4104b5cd Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 Apr 2024 15:03:00 -0700 Subject: [PATCH 050/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * AMReX: Weekly Update * pyAMReX: Weekly Update * Silence Parser Warning ``` Source/Utils/Parser/ParserUtils.cpp:177:31: error: ‘v’ may be used uninitialized in this function [-Werror=maybe-uninitialized] 177 | parser.setConstant(*it, v); | ~~~~~~~~~~~~~~~~~~^~~~~~~~ ``` --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- Source/Utils/Parser/ParserUtils.cpp | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index bfcd0d3e44b..f47dd01e057 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.04 && cd - + cd ../amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 65f3de6c43a..d1c98b315a0 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.04 +branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index b4db494516d..2f7fff84be7 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.04 +branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index c2da9577947..0339b766e38 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -164,7 +164,7 @@ amrex::Parser utils::parser::makeParser ( for (auto it = symbols.begin(); it != symbols.end(); ) { // Always parsing in double precision avoids potential overflows that may occur when parsing // user's expressions because of the limited range of exponentials in single precision - double v; + double v = 0.0; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( recursive_symbols.count(*it)==0, diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index aa7c4ead995..3f2987ec9cd 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.04" +set(WarpX_amrex_branch "2a3955a5f5aac1aef6e6e72687f182331e049c39" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index cc43b9e3a29..d4c488f9113 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.04" +set(WarpX_pyamrex_branch "9b0a4d82bc14756530efe4e0c9ce3463ca5df742" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 400ccf1737c..1d95dd5f2d2 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.04 && cd - +cd amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 76d65006798d7a70539d1f3b959d2a5038dd63a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:19:43 -0700 Subject: [PATCH 051/121] [pre-commit.ci] pre-commit autoupdate (#4837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 812a046b602..cef24aed2cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] From 9f5be94cd0b99063efcea7e568d438a53dc2df5b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 Apr 2024 15:34:35 -0700 Subject: [PATCH 052/121] HPC3 (UCI): Fix ADIOS2 HDF5 Build (#4836) Disable building examples and tests for ADIOS2 for speed. Do not build HDF5 bindings of ADIOS2 due an incompatibility in this version. --- Tools/machines/hpc3-uci/install_gpu_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index a2b9c7b3855..51d4bcfe2c5 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -70,7 +70,7 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/adios2-pm-gpu-build From b4ed7064f84a87e7ae9e133f21b3bd29fb7158b1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 8 Apr 2024 15:35:29 -0700 Subject: [PATCH 053/121] Output the potential phi on the macroparticles (#4599) * Output the potential phi on the macroparticles * Fix compilation * Cleaner code by using `getParticlePosition` * Add include statement * Update const-ness * Update code so that `phi` can be calculated * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Better boundary error message * Define function to define tiles, for a NamedComponentParticleContainer * Call parent class in WarpXParticleContainer * Call `defineAllParticleTiles` in diagnostics * Fix compilation issue * Fix compilation for fixed precision * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add automated test * Fix analysis script * Update test so that they work correctly in RZ * Update automated test * Update implementation of `phi` on the particles * Update test so as to trigger output of `phi` * Update documentation * Update Source/Diagnostics/ParticleDiag/ParticleDiag.H * Apply suggestions from code review Co-authored-by: Axel Huebl --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- Docs/source/usage/parameters.rst | 3 +- .../analysis_electrostatic_sphere.py | 18 ++- Examples/Tests/electrostatic_sphere/inputs_3d | 8 +- Examples/Tests/electrostatic_sphere/inputs_rz | 9 +- Regression/WarpX-tests.ini | 2 +- .../Diagnostics/ParticleDiag/ParticleDiag.H | 1 + .../Diagnostics/ParticleDiag/ParticleDiag.cpp | 19 ++- Source/Diagnostics/WarpXOpenPMD.cpp | 117 +++++++++++------- Source/Particles/ParticleIO.H | 46 +++++++ 9 files changed, 167 insertions(+), 56 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4ea9a3dc65d..0a089358d21 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2676,7 +2676,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``..variables`` (list of `strings` separated by spaces, optional) List of particle quantities to write to output. Choices are ``w`` for the particle weight and ``ux`` ``uy`` ``uz`` for the particle momenta. - By default, all particle quantities are written. + When using the lab-frame electrostatic solver, ``phi`` (electrostatic potential, on the macroparticles) is also available. + By default, all particle quantities (except ``phi``) are written. If ``..variables = none``, no particle data are written, except for particle positions, which are always included. * ``..random_fraction`` (`float`) optional diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index 35281084c01..8e05bfa0c68 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -22,6 +22,8 @@ import numpy as np import yt +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c from scipy.optimize import fsolve sys.path.insert(1, '../../../../warpx/Regression/Checksum/') @@ -125,7 +127,6 @@ def calculate_error(E_axis, xmin, dx, nx): return L2_error - L2_error_x = calculate_error(Ex_axis, xmin, dx, nx) L2_error_y = calculate_error(Ey_axis, ymin, dy, ny) L2_error_z = calculate_error(Ez_axis, zmin, dz, nz) @@ -137,6 +138,21 @@ def calculate_error(E_axis, xmin, dx, nx): assert L2_error_y < 0.05 assert L2_error_z < 0.05 +# Check conservation of energy +def return_energies(iteration): + ux, uy, uz, phi, m, q, w = ts.get_particle(['ux', 'uy', 'uz', 'phi', 'mass', 'charge', 'w'], iteration=iteration) + E_kinetic = (w*m*c**2 * (np.sqrt(1 + ux**2 + uy**2 + uz**2) - 1)).sum() + E_potential = 0.5*(w*q*phi).sum() # potential energy of particles in their own space-charge field: includes factor 1/2 + return E_kinetic, E_potential +ts = OpenPMDTimeSeries('./diags/diag2') +if 'phi' in ts.avail_record_components['electron']: + # phi is only available when this script is run with the labframe poisson solver + print('Checking conservation of energy') + Ek_i, Ep_i = return_energies(0) + Ek_f, Ep_f = return_energies(30) + assert Ep_f < 0.7*Ep_i # Check that potential energy changes significantly + assert abs( (Ek_i + Ep_i) - (Ek_f + Ep_f) ) < 0.003 * (Ek_i + Ep_i) # Check conservation of energy + # Checksum regression analysis test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/electrostatic_sphere/inputs_3d b/Examples/Tests/electrostatic_sphere/inputs_3d index afc6649f012..1f15b2da0de 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_3d +++ b/Examples/Tests/electrostatic_sphere/inputs_3d @@ -29,7 +29,13 @@ electron.profile = parse_density_function electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" electron.momentum_distribution_type = at_rest -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 diag2 + diag1.intervals = 30 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere/inputs_rz b/Examples/Tests/electrostatic_sphere/inputs_rz index 58f31046d89..26d8b0c7afc 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_rz +++ b/Examples/Tests/electrostatic_sphere/inputs_rz @@ -30,7 +30,14 @@ electron.profile = parse_density_function electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" electron.momentum_distribution_type = at_rest -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 diag2 + diag1.intervals = 30 diag1.diag_type = Full diag1.fields_to_plot = Er Et Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.electron.variables = ux uy uz w phi +diag2.format = openpmd diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 2f7fff84be7..12fed1e8bb9 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -554,7 +554,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w phi dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.H b/Source/Diagnostics/ParticleDiag/ParticleDiag.H index e9f54d2e5e1..45bc596a78e 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.H +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.H @@ -23,6 +23,7 @@ public: [[nodiscard]] PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } [[nodiscard]] std::string getSpeciesName() const { return m_name; } amrex::Vector m_plot_flags; + bool m_plot_phi = false; // Whether to output the potential phi on the particles bool m_do_random_filter = false; bool m_do_uniform_filter = false; diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 9954fa24285..7ed64ccf8ec 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -35,12 +35,19 @@ ParticleDiag::ParticleDiag( if (variables[0] != "none"){ const std::map existing_variable_names = pc->getParticleComps(); for (const auto& var : variables){ - const auto search = existing_variable_names.find(var); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - search != existing_variable_names.end(), - "variables argument '" + var - +"' is not an existing attribute for this species"); - m_plot_flags[existing_variable_names.at(var)] = 1; + if (var == "phi") { + // User requests phi on particle. This is *not* part of the variables that + // the particle container carries, and is only added to particles during output. + // Therefore, this case needs to be treated specifically. + m_plot_phi = true; + } else { + const auto search = existing_variable_names.find(var); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + search != existing_variable_names.end(), + "variables argument '" + var + +"' is not an existing attribute for this species"); + m_plot_flags[existing_variable_names.at(var)] = 1; + } } } } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index d90f3b45c96..6882b962445 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -548,48 +548,6 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { pinned_pc->make_alike() : pc->make_alike(); - // names of amrex::Real and int particle attributes in SoA data - amrex::Vector real_names; - amrex::Vector int_names; - amrex::Vector int_flags; - amrex::Vector real_flags; - - // see openPMD ED-PIC extension for namings - // note: an underscore separates the record name from its component - // for non-scalar records - // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX -#if !defined (WARPX_DIM_1D_Z) - real_names.push_back("position_x"); -#endif -#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - real_names.push_back("position_y"); -#endif - real_names.push_back("position_z"); - real_names.push_back("weighting"); - real_names.push_back("momentum_x"); - real_names.push_back("momentum_y"); - real_names.push_back("momentum_z"); - - // get the names of the real comps - real_names.resize(tmp.NumRealComps()); - auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) - { - real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); - } - // plot any "extra" fields by default - real_flags = particle_diags[i].m_plot_flags; - real_flags.resize(tmp.NumRealComps(), 1); - // and the names - int_names.resize(tmp.NumIntComps()); - auto runtime_inames = tmp.getParticleRuntimeiComps(); - for (auto const& x : runtime_inames) - { - int_names[x.second+0] = detail::snakeToCamel(x.first); - } - // plot by default - int_flags.resize(tmp.NumIntComps(), 1); - const auto mass = pc->AmIA() ? PhysConst::m_e : pc->getMass(); RandomFilter const random_filter(particle_diags[i].m_do_random_filter, particle_diags[i].m_random_fraction); @@ -631,6 +589,76 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); } + // Gather the electrostatic potential (phi) on the macroparticles + if ( particle_diags[i].m_plot_phi ) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || + (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); + // Using pinned PC indicates that the particles are not written at the same physical time (i.e. PIC iteration) + // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). + // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. + // To avoid confusion, we raise an error in this case. + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + use_pinned_pc == false, + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available with `diag_type = Full`."); + tmp.AddRealComp("phi"); + int const phi_index = tmp.getParticleComps().at("phi"); + auto& warpx = WarpX::GetInstance(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (int lev=0; lev<=warpx.finestLevel(); lev++) { + const amrex::Geometry& geom = warpx.Geom(lev); + auto plo = geom.ProbLoArray(); + auto dxi = geom.InvCellSizeArray(); + amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + storeFieldOnParticles( tmp, plo, dxi, phi, phi_index, lev ); + } + } + + // names of amrex::Real and int particle attributes in SoA data + amrex::Vector real_names; + amrex::Vector int_names; + amrex::Vector int_flags; + amrex::Vector real_flags; + // see openPMD ED-PIC extension for namings + // note: an underscore separates the record name from its component + // for non-scalar records + // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX +#if !defined (WARPX_DIM_1D_Z) + real_names.push_back("position_x"); +#endif +#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + real_names.push_back("position_y"); +#endif + real_names.push_back("position_z"); + real_names.push_back("weighting"); + real_names.push_back("momentum_x"); + real_names.push_back("momentum_y"); + real_names.push_back("momentum_z"); + // get the names of the real comps + real_names.resize(tmp.NumRealComps()); + auto runtime_rnames = tmp.getParticleRuntimeComps(); + for (auto const& x : runtime_rnames) + { + real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); + } + // plot any "extra" fields by default + real_flags = particle_diags[i].m_plot_flags; + real_flags.resize(tmp.NumRealComps(), 1); + // and the names + int_names.resize(tmp.NumIntComps()); + auto runtime_inames = tmp.getParticleRuntimeiComps(); + for (auto const& x : runtime_inames) + { + int_names[x.second+0] = detail::snakeToCamel(x.first); + } + // plot by default + int_flags.resize(tmp.NumIntComps(), 1); + // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. DumpToFile(&tmp, @@ -640,9 +668,8 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { int_flags, real_names, int_names, pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush - ); -} + isBTD, isLastBTDFlush); + } } void diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index c417f7467b3..1abda6ea9af 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -8,6 +8,9 @@ #define PARTICLEIO_H_ #include "Particles/WarpXParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" + +#include #include #include @@ -77,4 +80,47 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer* } } +/** Gathers the field from a MultiFab to the macroparticles + * and stores it into a runtime component of the particle container + * + * @tparam T_ParticleContainer a WarpX particle container or AmrParticleContainer + * @param tmp the particle container on which to store the gathered field + * @param plo the coordinates of the low bound of the box, along each dimension + * @param dxi the inverse of the cell size, along each dimension + * @param phi a multifab that contains the field to be gathered + * @param phi_index the index of the runtime component where to store the gathered field + * @param lev the MR level + */ +template< typename T_ParticleContainer > +void +storeFieldOnParticles ( T_ParticleContainer& tmp, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + amrex::MultiFab const& phi, + int const phi_index, int const lev ) { + + using PinnedParIter = typename T_ParticleContainer::ParIterType; + + for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { + + auto phi_grid = phi[pti].array(); + const auto getPosition = GetParticlePosition(pti); + amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); + + // Loop over the particles and update their position + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long ip) { + + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); + phi_particle_arr[ip] = phi_value; + } + ); + } +} + #endif /* PARTICLEIO_H_ */ From 56a00e947722cc392ea9e4f4bf81746789562ac2 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 9 Apr 2024 03:39:45 +0200 Subject: [PATCH 054/121] Remove calls to WarpX::GetInstance() in PEC class and cleaning (#4771) * remove calls to WarpX instance in PEC class * continue work * fixed bugs * complete removal of warpx instance in WarpX_PEC * update doc * default for field boundary is PML * add missing :: * fixed bug * fix bugs * make notation more compact * use isAnyBoundary function also for SilverMuller --- Source/BoundaryConditions/PML.H | 4 +- Source/BoundaryConditions/PML_RZ.H | 4 +- Source/BoundaryConditions/PML_fwd.H | 2 - .../WarpXFieldBoundaries.cpp | 132 +++- Source/BoundaryConditions/WarpX_PEC.H | 582 +++------------- Source/BoundaryConditions/WarpX_PEC.cpp | 631 +++++++++++++++--- .../ApplySilverMuellerBoundary.cpp | 4 +- .../FiniteDifferenceSolver.H | 5 +- .../HybridPICModel/HybridPICModel.H | 4 +- Source/Utils/WarpXAlgorithmSelection.H | 18 +- Source/Utils/WarpXAlgorithmSelection.cpp | 16 +- Source/WarpX.H | 10 +- Source/WarpX.cpp | 4 +- 13 files changed, 741 insertions(+), 675 deletions(-) diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 5f41ecbd706..835877f2de9 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -11,6 +11,8 @@ #include "PML_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif @@ -131,8 +133,6 @@ private: amrex::Real dt_E = -1.e10; }; -enum struct PatchType : int; - class PML { public: diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 3a5a51770f8..74d3da2e19b 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -10,6 +10,8 @@ #include "PML_RZ_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif @@ -25,8 +27,6 @@ #include #include -enum struct PatchType : int; - class PML_RZ { public: diff --git a/Source/BoundaryConditions/PML_fwd.H b/Source/BoundaryConditions/PML_fwd.H index 58532a89070..21ad81949f1 100644 --- a/Source/BoundaryConditions/PML_fwd.H +++ b/Source/BoundaryConditions/PML_fwd.H @@ -14,8 +14,6 @@ struct SigmaBox; class SigmaBoxFactory; class MultiSigmaBox; -enum struct PatchType; - class PML; #endif /* WARPX_PML_FWD_H */ diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index ebf3d1e39eb..0e50c560e7c 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -18,26 +18,72 @@ using namespace amrex::literals; using namespace amrex; +namespace +{ + /** Returns true if any field boundary is set to FieldBoundaryType FT, else returns false.*/ + template + [[nodiscard]] + bool isAnyBoundary (const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi) + { + const auto isFT = [](const auto& b){ + return b == FT;}; + return std::any_of(field_boundary_lo.begin(), field_boundary_lo.end(), isFT) || + std::any_of(field_boundary_hi.begin(), field_boundary_hi.end(), isFT); + } + + /** Returns true if any particle boundary is set to ParticleBoundaryType PT, else returns false.*/ + template + [[nodiscard]] + bool isAnyBoundary (const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi) + { + const auto isPT = [](const auto& b){ + return b == PT;}; + return std::any_of(particle_boundary_lo.begin(), particle_boundary_lo.end(), isPT) || + std::any_of(particle_boundary_hi.begin(), particle_boundary_hi.end(), isPT); + } + +} + void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2) }, lev, patch_type); - if (WarpX::isAnyBoundaryPML()) { + PEC::ApplyPECtoEfield( + {getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( pml[lev]->GetE_fp(), lev, patch_type, split_pml_field); + PEC::ApplyPECtoEfield( + pml[lev]->GetE_fp(), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); } } else { - PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_cp, lev, 0), - getFieldPointer(FieldType::Efield_cp, lev, 1), - getFieldPointer(FieldType::Efield_cp, lev, 2) }, lev, patch_type); - if (WarpX::isAnyBoundaryPML()) { + PEC::ApplyPECtoEfield( + {getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( pml[lev]->GetE_cp(), lev, patch_type, split_pml_field); + PEC::ApplyPECtoEfield( + pml[lev]->GetE_cp(), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); } } } @@ -57,15 +103,23 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_fp, lev, 0), - getFieldPointer(FieldType::Bfield_fp, lev, 1), - getFieldPointer(FieldType::Bfield_fp, lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { + getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2) }, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); } else { - PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_cp, lev, 0), - getFieldPointer(FieldType::Bfield_cp, lev, 1), - getFieldPointer(FieldType::Bfield_cp, lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { + getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); } } @@ -74,18 +128,11 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d // E and B are staggered in time, which is only true after the first half-push if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { - bool applySilverMueller = false; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( (WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller) || - (WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller) ) { - applySilverMueller = true; - } - } - if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp[lev], Bfield_fp[lev], - Geom(lev).Domain(), dt[lev], - WarpX::field_boundary_lo, - WarpX::field_boundary_hi); + if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ + m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + Efield_fp[lev], Bfield_fp[lev], + Geom(lev).Domain(), dt[lev], + field_boundary_lo, field_boundary_hi); } } } @@ -106,9 +153,14 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { - PEC::ApplyReflectiveBoundarytoRhofield(rho, lev, patch_type); + PEC::ApplyReflectiveBoundarytoRhofield(rho, + field_boundary_lo, field_boundary_hi, + particle_boundary_lo, particle_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } } @@ -116,9 +168,14 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { - PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, lev, patch_type); + PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, + field_boundary_lo, field_boundary_hi, + particle_boundary_lo, particle_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } } @@ -207,11 +264,12 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoElectronPressure( - m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), lev, patch_type - ); + m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), + field_boundary_lo, field_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } else { amrex::Abort(Utils::TextMsg::Err( "ApplyElectronPressureBoundary: Only one level implemented for hybrid solver.")); diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index b0d15a70321..a6af894beb4 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -1,16 +1,11 @@ #ifndef WARPX_PEC_KERNELS_H_ #define WARPX_PEC_KERNELS_H_ -#include "WarpX.H" #include "Utils/WarpXAlgorithmSelection.H" #include -#include -#include -#include -#include -#include -#include +#include +#include #include @@ -18,528 +13,113 @@ #include namespace PEC { -using namespace amrex; - /** - * \brief Determines if the field boundary condition stored in fboundary - * in direction, dir, is PEC. - * - * \param[in] fboundary Value containing boundary type - * \param[in] dir direction - * - * \returns 1 if the boundary type is PEC else 0 - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - bool is_boundary_PEC (amrex::GpuArray const& fboundary, int dir) { - return ( fboundary[dir] == FieldBoundaryType::PEC ); - } - - /** - * \brief Determines if the particle boundary condition stored in pboundary - * in direction, dir, is reflecting. - * - * \param[in] pboundary Value containing boundary type - * \param[in] dir direction - * - * \returns 1 if the boundary type is reflecting else 0 - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - bool is_boundary_reflecting (amrex::GpuArray const& pboundary, int dir) { - return ( pboundary[dir] == ParticleBoundaryType::Reflecting ); - } - - /** - * \brief Calculates the number of grid points the given index is pass the - * domain boundary i.e. a value of +1 means the current cell is - * outside of the simulation domain by 1 cell. Note that the high - * side domain boundary is between cell dom_hi and dom_hi+1 for cell - * centered grids and on cell dom_hi+1 for nodal grid. This is why - * (dom_hi[idim] + is_nodal[idim]) is used below. - * - * \param[in] dom_lo, dom_hi Domain boundaries - * \param[in] ijk_vec Cell coordinates - * \param[in] is_nodal Whether the field of interest is nodal - * \param[in] idim Dimension of interest - * \param[in] iside 0 for low and 1 for high - * - * \returns number of grid points to the boundary - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - int get_cell_count_to_boundary (const amrex::IntVect& dom_lo, - const amrex::IntVect& dom_hi, const amrex::IntVect& ijk_vec, - const amrex::IntVect& is_nodal, const int idim, const int iside) - { - return ((iside == 0) ? (dom_lo[idim] - ijk_vec[idim]) - : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); - } - - /** - * \brief Sets the electric field value tangential to the PEC boundary to zero. The - * tangential Efield components in the guard cells outside the - * domain boundary are set equal and opposite to the field in the valid cells - * at their mirrored locations. The normal Efield components in the guard cells - * are set equal to the field in the valid cells at their mirrored locations. - * The number or depth of guard cells updated is equal to the shape factor of - * particles in each dimension. - * For corner cells with mixed boundaries, the mirror location could be outside - * valid region, while still ensuring PEC condition is maintained across the - * PEC boundary, and the necessary sign change is accounted for depending on - * if the component, icomp, is tangential or normal to the PEC boundary. - * - * For 3D : - * x component is tangential to the y-boundary and z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary and y-boundary - * x component is normal to the x-boundary - * y component is normal to the y-boundary - * z component is normal to the z-boundary - * where, x-boundary is the yz-plane at x=xmin and x=xmax - * y-boundary is the xz-plane at y=ymin and y=ymax - * z-boundary is the xy-plane at z=zmin and z=zmax - * - * For 2D : WarpX uses X-Z as the two dimensions - * x component is tangential to the z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary - * x component is normal to the x-boundary - * y component is not normal to any boundary (Only xz dimensions in 2D) - * z component is normal to the z-boundary - * where, x-boundary is along the line z at x=xmin and x=xmax - * z-boundary is along the line x at z=zmin and z=zmax - * - * For 1D : WarpX uses Z as the only dimension - * x component is tangential to the z-boundary - * y component is tangential to the z-boundary - * z component is not tangential to the z-boundary - * x component is not normal to any boundary (Only z dimension in 1D) - * y component is not normal to any boundary (Only z dimension in 1D) - * z component is normal to the z-boundary - * where, z-boundary is a point at z=zmin and z=zmax - * - * For RZ : WarpX uses R-Z as the two dimensions - * r component is tangential to the z-boundary - * theta_component is tangential to the r-boundary and z-boundary - * z component is tangential to the r-boundary - * r component is normal to the r-boundary - * theta_component is not normal to any boundary (on RZ dimensions are modeled) - * z component is normal to the z-boundary - * where, r-boundary is along the line z at r=rmin and r=rmax - * z-boundary is along the line r at z=zmin and z=zmax - * - * - * \param[in] icomp component of the Efield being updated - * (0=x, 1=y, 2=z in Cartesian) - * (0=r, 1=theta, 2=z in RZ) - * \param[in] dom_lo index value of the lower domain boundary (cell-centered) - * \param[in] dom_hi index value of the higher domain boundary (cell-centered) - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 - * \param[in] n index of the MultiFab component being updated - * \param[in] Efield field data to be updated if (ijk) is at the boundary or a guard cell - * \param[in] is_nodal staggering of the field data being updated. - * \param[in] fbndry_lo Field boundary type at the lower boundaries - * \param[in] fbndry_hi Field boundary type at the upper boundaries - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetEfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, - const amrex::IntVect &dom_hi, - const amrex::IntVect &ijk_vec, const int n, - amrex::Array4 const& Efield, - const amrex::IntVect& is_nodal, - amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) - { - // Tangential Efield components in guard cells set equal and opposite to cells - // in the mirror locations across the PEC boundary, whereas normal E-field - // components are set equal to values in the mirror locations across the PEC - // boundary. Here we just initialize it. - amrex::IntVect ijk_mirror = ijk_vec; - bool OnPECBoundary = false; - bool GuardCell = false; - amrex::Real sign = 1._rt; - // Loop over all the dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Loop over sides, iside = 0 (lo), iside = 1 (hi) - for (int iside = 0; iside < 2; ++iside) { - const bool isPECBoundary = ( (iside == 0) - ? is_boundary_PEC(fbndry_lo, idim) - : is_boundary_PEC(fbndry_hi, idim) ); -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), - // icomp=1 is tangential to both x and z boundaries - // The logic below ensures that the flags are set right for 2D - const bool is_tangent_to_PEC = (icomp != AMREX_SPACEDIM*idim); -#elif (defined WARPX_DIM_1D_Z) - // For 1D : icomp=0 and icomp=1 (Ex and Ey are tangential to the z boundary) - // The logic below ensures that the flags are set right for 1D - const bool is_tangent_to_PEC = (icomp != idim+2); -#else - const bool is_tangent_to_PEC = (icomp != idim); -#endif - if (isPECBoundary) { - // grid point ijk_vec is ig number of points pass the - // domain boundary in direction, idim - const int ig = get_cell_count_to_boundary( - dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); - - if (ig == 0) { - if (is_tangent_to_PEC && is_nodal[idim] == 1) { - OnPECBoundary = true; - } - } else if (ig > 0) { - // Find mirror location across PEC boundary - ijk_mirror[idim] = ( ( iside == 0) - ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) - : (dom_hi[idim] + 1 - ig)); - GuardCell = true; - // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) { sign *= -1._rt; } -#if (defined WARPX_DIM_RZ) - if (icomp == 0 && idim == 0 && iside == 1) { - // Add radial scale so that drEr/dr = 0. - // This only works for the first guard cell and with - // Er cell centered in r. - const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - sign *= rmirror/rguard; - } -#endif - } - } // is PEC boundary - } // loop over iside - } // loop over dimensions - if (OnPECBoundary) { - // if ijk_vec is on a PEC boundary in any direction, set Etangential to 0. - Efield(ijk_vec,n) = 0._rt; - } else if (GuardCell) { - Efield(ijk_vec,n) = sign * Efield(ijk_mirror,n); - } - } - - - /** - * \brief Sets the magnetic field value normal to the PEC boundary to zero. The - * tangential (and normal) field value of the guard cells outside the - * domain boundary are set equal (and opposite) to the respective field components - * in the valid cells at their mirrored locations. - * The number or depth of guard cells updated is equal to the shape factor of - * particles in each dimension. - * - * For 3D : - * x component is tangential to the y-boundary and z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary and y-boundary - * x component is normal to the x-boundary - * y component is normal to the y-boundary - * z component is normal to the z-boundary - * where, x-boundary is the yz-plane at x=xmin and x=xmax - * y-boundary is the xz-plane at y=ymin and y=ymax - * z-boundary is the xy-plane at z=zmin and z=zmax - * - * For 2D : WarpX uses X-Z as the two dimensions - * x component is tangential to the z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary - * x component is normal to the x-boundary - * y component is not normal to any boundary (Only xz dimensions in 2D) - * z component is normal to the z-boundary - * where, x-boundary is along the line z at x=xmin and x=xmax - * z-boundary is along the line x at z=zmin and z=zmax - * - * For 1D : WarpX uses Z as the only dimension - * x component is tangential to the z-boundary - * y component is tangential to the z-boundary - * z component is not tangential to the z-boundary - * x component is not normal to any boundary (Only z dimension in 1D) - * y component is not normal to any boundary (Only z dimension in 1D) - * z component is normal to the z-boundary - * where, z-boundary is a point at z=zmin and z=zmax - * - * For RZ : WarpX uses R-Z as the two dimensions - * r component is tangential to the z-boundary - * theta_component is tangential to the r-boundary and z-boundary - * z component is tangential to the r-boundary - * r component is normal to the r-boundary - * theta_component is not normal to any boundary (on RZ dimensions are modeled) - * z component is normal to the z-boundary - * where, r-boundary is along the line z at r=rmin and r=rmax - * z-boundary is along the line r at z=zmin and z=zmax - * - * - * \param[in] icomp component of the Bfield being updated - * (0=x, 1=y, 2=z in Cartesian) - * (0=r, 1=theta, 2=z in RZ) - * \param[in] dom_lo index value of the lower domain boundary (cell-centered) - * \param[in] dom_hi index value of the higher domain boundary (cell-centered) - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 - * \param[in] n index of the MultiFab component being updated - * \param[in] Bfield field data to be updated if (ijk) is at the boundary - or a guard cell - * \param[in] is_nodal staggering of the field data being updated. - * \param[in] fbndry_lo Field boundary type at the lower boundaries - * \param[in] fbndry_hi Field boundary type at the upper boundaries - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetBfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, - const amrex::IntVect & dom_hi, - const amrex::IntVect & ijk_vec, const int n, - amrex::Array4 const& Bfield, - const amrex::IntVect & is_nodal, - amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) - { - amrex::IntVect ijk_mirror = ijk_vec; - bool OnPECBoundary = false; - bool GuardCell = false; - amrex::Real sign = 1._rt; - // Loop over all dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Loop over sides, iside = 0 (lo), iside = 1 (hi) - for (int iside = 0; iside < 2; ++iside) { - const bool isPECBoundary = ( (iside == 0 ) - ? is_boundary_PEC(fbndry_lo, idim) - : is_boundary_PEC(fbndry_hi, idim) ); - if (isPECBoundary) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - // For 2D : for icomp==1, (By in XZ, Btheta in RZ), - // icomp=1 is not normal to x or z boundary - // The logic below ensures that the flags are set right for 2D - const bool is_normal_to_PEC = (icomp == (AMREX_SPACEDIM*idim)); -#elif (defined WARPX_DIM_1D_Z) - // For 1D : icomp=0 and icomp=1 (Bx and By are not normal to the z boundary) - // The logic below ensures that the flags are set right for 1D - const bool is_normal_to_PEC = (icomp == (idim+2)); -#else - const bool is_normal_to_PEC = (icomp == idim); -#endif - - // grid point ijk_vec is ig number of points pass the - // domain boundary in direction, idim - const int ig = get_cell_count_to_boundary( - dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); - - if (ig == 0) { - // Only normal component is set to 0 - if (is_normal_to_PEC && is_nodal[idim]==1) { - OnPECBoundary = true; - } - } else if ( ig > 0) { - // Mirror location inside the domain by "ig" number of cells - // across PEC boundary in direction, idim, and side, iside - ijk_mirror[idim] = ( (iside == 0) - ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) - : (dom_hi[idim] + 1 - ig)); - GuardCell = true; - // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) { sign *= -1._rt; } -#if (defined WARPX_DIM_RZ) - if (icomp == 0 && idim == 0 && iside == 1) { - // Add radial scale so that drBr/dr = 0. - const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - sign *= rmirror/rguard; - } -#endif - } - } // if PEC Boundary - } // loop over sides - } // loop of dimensions - - if (OnPECBoundary) { - // if ijk_vec is on a PEC boundary in any direction, set Bnormal to 0. - Bfield(ijk_vec,n) = 0._rt; - } else if (GuardCell) { - // Bnormal and Btangential is set opposite and equal to the value - // in the mirror location, respectively. - Bfield(ijk_vec,n) = sign * Bfield(ijk_mirror,n); - } - } - - - /** - * \brief Sets the rho or J field value in cells close to and on reflecting particle boundary - * or PEC field boundary. The charge/current density deposited - * in the guard cells are either reflected - * back into the simulation domain (if a reflecting particle - * boundary is used), or the opposite charge/current density is deposited - * back in the domain to capture the effect of an image charge. - * The charge/current density on the reflecting boundary is set to 0 while values - * in the guard cells are set equal (and opposite) to their mirror - * location inside the domain - representing image charges - in the - * normal (tangential) direction. - * - * \param[in] n index of the MultiFab component being updated - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 - * \param[in out] field field data to be updated - * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec - * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_reflective Whether the given particle boundary is reflecting or field boundary is pec - * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary - * \param[in] fabbox multifab box including ghost cells - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetRhoOrJfieldFromPEC (const int n, - const amrex::IntVect & ijk_vec, - amrex::Array4 const& field, - amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, - amrex::GpuArray, AMREX_SPACEDIM> const& psign, - amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, - amrex::GpuArray const& tangent_to_bndy, - amrex::Box const& fabbox) - { - // The boundary is handled in 2 steps: - // 1) The cells internal to the domain are updated using the - // current deposited in the guard cells - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_reflective[idim][iside]) { continue; } - - // Get the mirror guard cell index - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - - // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) { - field(ijk_vec, n) = 0._rt; - // otherwise update the internal cell if the mirror guard cell exists - } else if (fabbox.contains(iv_mirror)) { - field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); - } - } - } - // 2) The guard cells are updated with the appropriate image - // charge based on the charge/current in the valid cells - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_reflective[idim][iside]) { continue; } - - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) - { - if (tangent_to_bndy[idim]) { - field(iv_mirror, n) = -field(ijk_vec, n); - } else { - field(iv_mirror, n) = field(ijk_vec, n); - } - } - } - } - } - - - /** - * \brief This function sets the given field value on a PEC boundary - * to enforce a Neumann boundary condition (zero derivative) in the - * normal direction. - * - * \param[in] n index of the MultiFab component being updated - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 - * \param[in out] field field data to be updated - * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec - * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_pec Whether the given boundary is PEC - * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary - * \param[in] fabbox multifab box including ghost cells - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetNeumannOnPEC (const int n, - const amrex::IntVect & ijk_vec, - amrex::Array4 const& field, - amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, - amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, - amrex::Box const& fabbox ) - { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_pec[idim][iside]) { continue; } - - // Get the mirror guard cell index - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - - // On the PEC boundary the field value is set equal to the - // first value in the domain (nodal fields) - if (ijk_vec == iv_mirror) { - iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } - } - // otherwise set the mirror guard cell equal to the internal cell value - else if (fabbox.contains(iv_mirror)) - { - field(iv_mirror, n) = field(ijk_vec, n); - } - } - } - } - - - /** Returns 1 if any domain boundary is set to PEC, else returns 0.*/ - bool isAnyBoundaryPEC(); - - /** Return true if any particle boundary is set to reflecting, else returns false*/ - bool isAnyParticleBoundaryReflecting (); /** * \brief Sets the tangential electric field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell * field value at the respective mirror locations. * - * \param[in,out] Efield Boundary values of tangential Efield are set to zero - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine - * \param[in] split_pml_field whether pml the multifab is the regular Efield or - * split pml field + * \param[in,out] Efield Boundary values of tangential Efield are set to zero + * \param[in] field_boundary_lo Boundary types of the "low" boundaries + * \param[in] field_boundary_hi Boundary types of the "high" boundaries + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field */ - void ApplyPECtoEfield ( std::array Efield, - int lev, PatchType patch_type, - bool split_pml_field = false); + void ApplyPECtoEfield ( + std::array Efield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + bool split_pml_field = false); /** * \brief Sets the normal component of the magnetic field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell * field value at the respective mirror locations. * - * \param[in,out] Bfield Boundary values of normal Bfield are set to zero. - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Bfield Boundary values of normal Bfield are set to zero. + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoBfield ( std::array Bfield, - int lev, PatchType patch_type); + void ApplyPECtoBfield ( + std::array Bfield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Reflects charge density deposited over the PEC boundary back into * the simulation domain. * - * \param[in,out] rho Multifab containing the charge density - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] rho Multifab containing the charge density + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] particle_boundary_lo Boundary types of the "low" particle boundaries + * \param[in] particle_boundary_hi Boundary types of the "high" particle boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyReflectiveBoundarytoRhofield(amrex::MultiFab* rho, int lev, - PatchType patch_type); + void ApplyReflectiveBoundarytoRhofield( + amrex::MultiFab* rho, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Reflects current density deposited over the PEC boundary back into * the simulation domain. * - * \param[in,out] Jx, Jy, Jz Multifabs containing the current density - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Jx, Jy, Jz Multifabs containing the current density + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] particle_boundary_lo Boundary types of the "low" particle boundaries + * \param[in] particle_boundary_hi Boundary types of the "high" particle boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, - amrex::MultiFab* Jz, int lev, - PatchType patch_type); + void ApplyReflectiveBoundarytoJfield( + amrex::MultiFab* Jx, amrex::MultiFab* Jy, + amrex::MultiFab* Jz, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Apply the PEC boundary to the electron pressure field. * - * \param[in,out] Pefield Multifab containing the electron pressure - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Pefield Multifab containing the electron pressure + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, - int lev, PatchType patch_type); + void ApplyPECtoElectronPressure ( + amrex::MultiFab* Pefield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); } #endif // WarpX_PEC_KERNELS_H_ diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index a3d92fa507d..a42873db555 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -1,60 +1,483 @@ #include "BoundaryConditions/WarpX_PEC.H" -#include "WarpX.H" #include +#include #include +#include +#include #include #include #include +#include #include +#include #include #include +#include #include -#include +using namespace amrex; using namespace amrex::literals; -bool -PEC::isAnyBoundaryPEC() { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } +namespace +{ + /** + * \brief Calculates the number of grid points the given index is pass the + * domain boundary i.e. a value of +1 means the current cell is + * outside of the simulation domain by 1 cell. Note that the high + * side domain boundary is between cell dom_hi and dom_hi+1 for cell + * centered grids and on cell dom_hi+1 for nodal grid. This is why + * (dom_hi[idim] + is_nodal[idim]) is used below. + * + * \param[in] dom_lo, dom_hi Domain boundaries + * \param[in] ijk_vec Cell coordinates + * \param[in] is_nodal Whether the field of interest is nodal + * \param[in] idim Dimension of interest + * \param[in] iside 0 for low and 1 for high + * + * \returns number of grid points to the boundary + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + int get_cell_count_to_boundary (const amrex::IntVect& dom_lo, + const amrex::IntVect& dom_hi, const amrex::IntVect& ijk_vec, + const amrex::IntVect& is_nodal, const int idim, const int iside) + { + return ((iside == 0) ? (dom_lo[idim] - ijk_vec[idim]) + : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); } - return false; -} -bool -PEC::isAnyParticleBoundaryReflecting () { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) {return true;} - if (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) {return true;} + + /** + * \brief Sets the electric field value tangential to the PEC boundary to zero. The + * tangential Efield components in the guard cells outside the + * domain boundary are set equal and opposite to the field in the valid cells + * at their mirrored locations. The normal Efield components in the guard cells + * are set equal to the field in the valid cells at their mirrored locations. + * The number or depth of guard cells updated is equal to the shape factor of + * particles in each dimension. + * For corner cells with mixed boundaries, the mirror location could be outside + * valid region, while still ensuring PEC condition is maintained across the + * PEC boundary, and the necessary sign change is accounted for depending on + * if the component, icomp, is tangential or normal to the PEC boundary. + * + * For 3D : + * x component is tangential to the y-boundary and z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary and y-boundary + * x component is normal to the x-boundary + * y component is normal to the y-boundary + * z component is normal to the z-boundary + * where, x-boundary is the yz-plane at x=xmin and x=xmax + * y-boundary is the xz-plane at y=ymin and y=ymax + * z-boundary is the xy-plane at z=zmin and z=zmax + * + * For 2D : WarpX uses X-Z as the two dimensions + * x component is tangential to the z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary + * x component is normal to the x-boundary + * y component is not normal to any boundary (Only xz dimensions in 2D) + * z component is normal to the z-boundary + * where, x-boundary is along the line z at x=xmin and x=xmax + * z-boundary is along the line x at z=zmin and z=zmax + * + * For 1D : WarpX uses Z as the only dimension + * x component is tangential to the z-boundary + * y component is tangential to the z-boundary + * z component is not tangential to the z-boundary + * x component is not normal to any boundary (Only z dimension in 1D) + * y component is not normal to any boundary (Only z dimension in 1D) + * z component is normal to the z-boundary + * where, z-boundary is a point at z=zmin and z=zmax + * + * For RZ : WarpX uses R-Z as the two dimensions + * r component is tangential to the z-boundary + * theta_component is tangential to the r-boundary and z-boundary + * z component is tangential to the r-boundary + * r component is normal to the r-boundary + * theta_component is not normal to any boundary (on RZ dimensions are modeled) + * z component is normal to the z-boundary + * where, r-boundary is along the line z at r=rmin and r=rmax + * z-boundary is along the line r at z=zmin and z=zmax + * + * + * \param[in] icomp component of the Efield being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] Efield field data to be updated if (ijk) is at the boundary or a guard cell + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] fbndry_lo Field boundary type at the lower boundaries + * \param[in] fbndry_hi Field boundary type at the upper boundaries + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetEfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, + const amrex::IntVect &dom_hi, + const amrex::IntVect &ijk_vec, const int n, + amrex::Array4 const& Efield, + const amrex::IntVect& is_nodal, + amrex::GpuArray const& fbndry_lo, + amrex::GpuArray const& fbndry_hi ) + { + // Tangential Efield components in guard cells set equal and opposite to cells + // in the mirror locations across the PEC boundary, whereas normal E-field + // components are set equal to values in the mirror locations across the PEC + // boundary. Here we just initialize it. + amrex::IntVect ijk_mirror = ijk_vec; + bool OnPECBoundary = false; + bool GuardCell = false; + amrex::Real sign = 1._rt; + // Loop over all the dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = 0 (lo), iside = 1 (hi) + for (int iside = 0; iside < 2; ++iside) { + const bool isPECBoundary = ( (iside == 0) + ? fbndry_lo[idim] == FieldBoundaryType::PEC + : fbndry_hi[idim] == FieldBoundaryType::PEC ); +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), + // icomp=1 is tangential to both x and z boundaries + // The logic below ensures that the flags are set right for 2D + const bool is_tangent_to_PEC = (icomp != AMREX_SPACEDIM*idim); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Ex and Ey are tangential to the z boundary) + // The logic below ensures that the flags are set right for 1D + const bool is_tangent_to_PEC = (icomp != idim+2); +#else + const bool is_tangent_to_PEC = (icomp != idim); +#endif + if (isPECBoundary) { + // grid point ijk_vec is ig number of points pass the + // domain boundary in direction, idim + const int ig = ::get_cell_count_to_boundary( + dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); + + if (ig == 0) { + if (is_tangent_to_PEC && is_nodal[idim] == 1) { + OnPECBoundary = true; + } + } else if (ig > 0) { + // Find mirror location across PEC boundary + ijk_mirror[idim] = ( ( iside == 0) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + GuardCell = true; + // tangential components are inverted across PEC boundary + if (is_tangent_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drEr/dr = 0. + // This only works for the first guard cell and with + // Er cell centered in r. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif + } + } // is PEC boundary + } // loop over iside + } // loop over dimensions + if (OnPECBoundary) { + // if ijk_vec is on a PEC boundary in any direction, set Etangential to 0. + Efield(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + Efield(ijk_vec,n) = sign * Efield(ijk_mirror,n); + } + } + + + /** + * \brief Sets the magnetic field value normal to the PEC boundary to zero. The + * tangential (and normal) field value of the guard cells outside the + * domain boundary are set equal (and opposite) to the respective field components + * in the valid cells at their mirrored locations. + * The number or depth of guard cells updated is equal to the shape factor of + * particles in each dimension. + * + * For 3D : + * x component is tangential to the y-boundary and z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary and y-boundary + * x component is normal to the x-boundary + * y component is normal to the y-boundary + * z component is normal to the z-boundary + * where, x-boundary is the yz-plane at x=xmin and x=xmax + * y-boundary is the xz-plane at y=ymin and y=ymax + * z-boundary is the xy-plane at z=zmin and z=zmax + * + * For 2D : WarpX uses X-Z as the two dimensions + * x component is tangential to the z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary + * x component is normal to the x-boundary + * y component is not normal to any boundary (Only xz dimensions in 2D) + * z component is normal to the z-boundary + * where, x-boundary is along the line z at x=xmin and x=xmax + * z-boundary is along the line x at z=zmin and z=zmax + * + * For 1D : WarpX uses Z as the only dimension + * x component is tangential to the z-boundary + * y component is tangential to the z-boundary + * z component is not tangential to the z-boundary + * x component is not normal to any boundary (Only z dimension in 1D) + * y component is not normal to any boundary (Only z dimension in 1D) + * z component is normal to the z-boundary + * where, z-boundary is a point at z=zmin and z=zmax + * + * For RZ : WarpX uses R-Z as the two dimensions + * r component is tangential to the z-boundary + * theta_component is tangential to the r-boundary and z-boundary + * z component is tangential to the r-boundary + * r component is normal to the r-boundary + * theta_component is not normal to any boundary (on RZ dimensions are modeled) + * z component is normal to the z-boundary + * where, r-boundary is along the line z at r=rmin and r=rmax + * z-boundary is along the line r at z=zmin and z=zmax + * + * + * \param[in] icomp component of the Bfield being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] Bfield field data to be updated if (ijk) is at the boundary + or a guard cell + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] fbndry_lo Field boundary type at the lower boundaries + * \param[in] fbndry_hi Field boundary type at the upper boundaries + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetBfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, + const amrex::IntVect & dom_hi, + const amrex::IntVect & ijk_vec, const int n, + amrex::Array4 const& Bfield, + const amrex::IntVect & is_nodal, + amrex::GpuArray const& fbndry_lo, + amrex::GpuArray const& fbndry_hi ) + { + amrex::IntVect ijk_mirror = ijk_vec; + bool OnPECBoundary = false; + bool GuardCell = false; + amrex::Real sign = 1._rt; + // Loop over all dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = 0 (lo), iside = 1 (hi) + for (int iside = 0; iside < 2; ++iside) { + const bool isPECBoundary = ( (iside == 0) + ? fbndry_lo[idim] == FieldBoundaryType::PEC + : fbndry_hi[idim] == FieldBoundaryType::PEC ); + if (isPECBoundary) { +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (By in XZ, Btheta in RZ), + // icomp=1 is not normal to x or z boundary + // The logic below ensures that the flags are set right for 2D + const bool is_normal_to_PEC = (icomp == (AMREX_SPACEDIM*idim)); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Bx and By are not normal to the z boundary) + // The logic below ensures that the flags are set right for 1D + const bool is_normal_to_PEC = (icomp == (idim+2)); +#else + const bool is_normal_to_PEC = (icomp == idim); +#endif + + // grid point ijk_vec is ig number of points pass the + // domain boundary in direction, idim + const int ig = ::get_cell_count_to_boundary( + dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); + + if (ig == 0) { + // Only normal component is set to 0 + if (is_normal_to_PEC && is_nodal[idim]==1) { + OnPECBoundary = true; + } + } else if ( ig > 0) { + // Mirror location inside the domain by "ig" number of cells + // across PEC boundary in direction, idim, and side, iside + ijk_mirror[idim] = ( (iside == 0) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + GuardCell = true; + // Sign of the normal component in guard cell is inverted + if (is_normal_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drBr/dr = 0. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif + } + } // if PEC Boundary + } // loop over sides + } // loop of dimensions + + if (OnPECBoundary) { + // if ijk_vec is on a PEC boundary in any direction, set Bnormal to 0. + Bfield(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + // Bnormal and Btangential is set opposite and equal to the value + // in the mirror location, respectively. + Bfield(ijk_vec,n) = sign * Bfield(ijk_mirror,n); + } + } + + + /** + * \brief Sets the rho or J field value in cells close to and on reflecting particle boundary + * or PEC field boundary. The charge/current density deposited + * in the guard cells are either reflected + * back into the simulation domain (if a reflecting particle + * boundary is used), or the opposite charge/current density is deposited + * back in the domain to capture the effect of an image charge. + * The charge/current density on the reflecting boundary is set to 0 while values + * in the guard cells are set equal (and opposite) to their mirror + * location inside the domain - representing image charges - in the + * normal (tangential) direction. + * + * \param[in] n index of the MultiFab component being updated + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 + * \param[in out] field field data to be updated + * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec + * \param[in] psign Whether the field value should be flipped across the boundary + * \param[in] is_reflective Whether the given particle boundary is reflecting or field boundary is pec + * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary + * \param[in] fabbox multifab box including ghost cells + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetRhoOrJfieldFromPEC (const int n, + const amrex::IntVect & ijk_vec, + amrex::Array4 const& field, + amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, + amrex::GpuArray, AMREX_SPACEDIM> const& psign, + amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, + amrex::GpuArray const& tangent_to_bndy, + amrex::Box const& fabbox) + { + // The boundary is handled in 2 steps: + // 1) The cells internal to the domain are updated using the + // current deposited in the guard cells + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_reflective[idim][iside]) { continue; } + + // Get the mirror guard cell index + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + + // On the PEC boundary the charge/current density is set to 0 + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; + // otherwise update the internal cell if the mirror guard cell exists + } else if (fabbox.contains(iv_mirror)) { + field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); + } + } + } + // 2) The guard cells are updated with the appropriate image + // charge based on the charge/current in the valid cells + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_reflective[idim][iside]) { continue; } + + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) + { + if (tangent_to_bndy[idim]) { + field(iv_mirror, n) = -field(ijk_vec, n); + } else { + field(iv_mirror, n) = field(ijk_vec, n); + } + } + } + } + } + + + /** + * \brief This function sets the given field value on a PEC boundary + * to enforce a Neumann boundary condition (zero derivative) in the + * normal direction. + * + * \param[in] n index of the MultiFab component being updated + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 + * \param[in out] field field data to be updated + * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec + * \param[in] psign Whether the field value should be flipped across the boundary + * \param[in] is_pec Whether the given boundary is PEC + * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary + * \param[in] fabbox multifab box including ghost cells + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetNeumannOnPEC (const int n, + const amrex::IntVect & ijk_vec, + amrex::Array4 const& field, + amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, + amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, + amrex::Box const& fabbox ) + { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_pec[idim][iside]) { continue; } + + // Get the mirror guard cell index + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + + // On the PEC boundary the field value is set equal to the + // first value in the domain (nodal fields) + if (ijk_vec == iv_mirror) { + iv_mirror[idim] += (iside == 0) ? 1 : -1; + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } + } + // otherwise set the mirror guard cell equal to the internal cell value + else if (fabbox.contains(iv_mirror)) + { + field(iv_mirror, n) = field(ijk_vec, n); + } + } + } } - return false; } void -PEC::ApplyPECtoEfield (std::array Efield, const int lev, - PatchType patch_type, const bool split_pml_field) +PEC::ApplyPECtoEfield ( + std::array Efield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + const bool split_pml_field) { - auto& warpx = WarpX::GetInstance(); - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } const amrex::IntVect domain_lo = domain_box.smallEnd(); const amrex::IntVect domain_hi = domain_box.bigEnd(); - amrex::GpuArray fbndry_lo; - amrex::GpuArray fbndry_hi; + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - fbndry_lo[idim] = WarpX::field_boundary_lo[idim]; - fbndry_hi[idim] = WarpX::field_boundary_hi[idim]; + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; } const amrex::IntVect Ex_nodal = Efield[0]->ixType().toIntVect(); const amrex::IntVect Ey_nodal = Efield[1]->ixType().toIntVect(); const amrex::IntVect Ez_nodal = Efield[2]->ixType().toIntVect(); - const amrex::IntVect ng_fieldgather = warpx.get_ng_fieldgather(); // For each Efield multifab, apply PEC boundary condition to ncomponents // If not split E-field, the PEC is applied to the regular Efield used in Maxwell's eq. // If split_pml_field is true, then PEC is applied to all the split field components of the tangential field. @@ -95,7 +518,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ex, Ex_nodal, fbndry_lo, fbndry_hi); }, tey, nComp_y, @@ -108,7 +531,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ey, Ey_nodal, fbndry_lo, fbndry_hi); }, tez, nComp_z, @@ -121,7 +544,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ez, Ez_nodal, fbndry_lo, fbndry_hi); } ); @@ -130,27 +553,28 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, void -PEC::ApplyPECtoBfield (std::array Bfield, const int lev, - PatchType patch_type) +PEC::ApplyPECtoBfield ( + std::array Bfield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } const amrex::IntVect domain_lo = domain_box.smallEnd(); const amrex::IntVect domain_hi = domain_box.bigEnd(); - amrex::GpuArray fbndry_lo; - amrex::GpuArray fbndry_hi; + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - fbndry_lo[idim] = WarpX::field_boundary_lo[idim]; - fbndry_hi[idim] = WarpX::field_boundary_hi[idim]; + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; } const amrex::IntVect Bx_nodal = Bfield[0]->ixType().toIntVect(); const amrex::IntVect By_nodal = Bfield[1]->ixType().toIntVect(); const amrex::IntVect Bz_nodal = Bfield[2]->ixType().toIntVect(); - const amrex::IntVect ng_fieldgather = warpx.get_ng_fieldgather(); const int nComp_x = Bfield[0]->nComp(); const int nComp_y = Bfield[1]->nComp(); const int nComp_z = Bfield[2]->nComp(); @@ -186,7 +610,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Bx, Bx_nodal, fbndry_lo, fbndry_hi); }, tby, nComp_y, @@ -199,7 +623,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, By, By_nodal, fbndry_lo, fbndry_hi); }, tbz, nComp_z, @@ -212,7 +636,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Bz, Bz_nodal, fbndry_lo, fbndry_hi); } ); @@ -231,14 +655,18 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, * location inside the domain - representing image charges. **/ void -PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_type) +PEC::ApplyReflectiveBoundarytoRhofield ( + amrex::MultiFab* rho, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } domain_box.convert(rho->ixType()); @@ -257,12 +685,12 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat amrex::GpuArray, AMREX_SPACEDIM> psign; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); - is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); + is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -270,11 +698,11 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat // components of the current density is_tangent_to_bndy[idim] = true; - psign[idim][0] = ( ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) + psign[idim][0] = ((particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; - psign[idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) + psign[idim][1] = ((particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - rho_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - rho_nodal[idim]); @@ -307,7 +735,7 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, rho_array, mirrorfac, psign, is_reflective, is_tangent_to_bndy, fabbox ); @@ -317,16 +745,18 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat void -PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, - amrex::MultiFab* Jz, const int lev, - PatchType patch_type) +PEC::ApplyReflectiveBoundarytoJfield( + amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } // Note: force domain box to be nodal to simplify the mirror cell @@ -355,12 +785,12 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::GpuArray, AMREX_SPACEDIM>, 3> psign; amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); - is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); + is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -381,19 +811,19 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, #endif if (is_tangent_to_bndy[icomp][idim]){ - psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; - psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; } else { - psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? -1._rt : 1._rt; - psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? -1._rt : 1._rt; } } @@ -437,7 +867,7 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, Jx_array, mirrorfac[0], psign[0], is_reflective, is_tangent_to_bndy[0], fabbox ); @@ -472,7 +902,7 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, Jy_array, mirrorfac[1], psign[1], is_reflective, is_tangent_to_bndy[1], fabbox ); @@ -507,7 +937,7 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, Jz_array, mirrorfac[2], psign[2], is_reflective, is_tangent_to_bndy[2], fabbox ); @@ -516,15 +946,16 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, } void -PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, - PatchType patch_type) +PEC::ApplyPECtoElectronPressure ( + amrex::MultiFab* Pefield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } domain_box.convert(Pefield->ixType()); @@ -541,8 +972,8 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, amrex::GpuArray, AMREX_SPACEDIM> is_pec; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; + is_pec[idim][0] = field_boundary_lo[idim] == FieldBoundaryType::PEC; + is_pec[idim][1] = field_boundary_hi[idim] == FieldBoundaryType::PEC; if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -577,7 +1008,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetNeumannOnPEC(n, iv, Pe_array, mirrorfac, is_pec, fabbox); + ::SetNeumannOnPEC(n, iv, Pe_array, mirrorfac, is_pec, fabbox); }); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 36be09696fd..5e75698903e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -39,8 +39,8 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real const dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi) { + amrex::Vector field_boundary_lo, + amrex::Vector field_boundary_hi) { // Ensure that we are using the Yee solver WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index d045a30dd44..a7a4f10a713 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -10,6 +10,7 @@ #include "EmbeddedBoundary/WarpXFaceInfoBox_fwd.H" #include "FiniteDifferenceSolver_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "BoundaryConditions/PML_fwd.H" #include "Evolve/WarpXDtType.H" @@ -89,8 +90,8 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi); + amrex::Vector field_boundary_lo, + amrex::Vector field_boundary_hi); void ComputeDivE ( const std::array,3>& Efield, amrex::MultiFab& divE ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 7d477b4e5fd..5c0c2bcc96a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -12,16 +12,16 @@ #include "HybridPICModel_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" -#include "WarpX.H" #include #include - /** * \brief This class contains the parameters needed to evaluate hybrid field * solutions (kinetic ions with fluid electrons). diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 1da869b4ba0..9768b36cb84 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -55,6 +55,12 @@ struct GridType { }; }; +enum struct PatchType +{ + fine, + coarse +}; + struct ElectromagneticSolverAlgo { enum { None = 0, @@ -146,14 +152,13 @@ struct LoadBalanceCostsUpdateAlgo { /** Field boundary conditions at the domain boundary */ -struct FieldBoundaryType { - enum { +enum struct FieldBoundaryType { PML = 0, Periodic = 1, - PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 + PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 PMC = 3, //!< perfect magnetic conductor (PMC) with B_tangential=0 Damped = 4, // Fields in the guard cells are damped for PSATD - //in the moving window direction + //in the moving window direction Absorbing_SilverMueller = 5, // Silver-Mueller boundary condition Neumann = 6, // For electrostatic, the normal E is set to zero None = 7, // The fields values at the boundary are not updated. This is @@ -161,7 +166,6 @@ struct FieldBoundaryType { Open = 8 // Used in the Integrated Green Function Poisson solver // Note that the solver implicitely assumes open BCs: // no need to enforce them separately - }; }; /** Particle boundary conditions at the domain boundary @@ -190,7 +194,7 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ); /** Select BC Type for fields, if field=true * else select BCType for particles. */ -int +FieldBoundaryType GetFieldBCTypeInteger( std::string BCType ); /** Select BC Type for particles. @@ -201,6 +205,6 @@ GetParticleBCTypeInteger( std::string BCType ); /** Find the name associated with a BC type */ std::string -GetFieldBCTypeString( int fb_type ); +GetFieldBCTypeString( FieldBoundaryType fb_type ); #endif // UTILS_WARPXALGORITHMSELECTION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 3386ce55b48..b1f93c2f6c8 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -123,7 +123,7 @@ const std::map MacroscopicSolver_algo_to_int = { {"default", MacroscopicSolverAlgo::BackwardEuler} }; -const std::map FieldBCType_algo_to_int = { +const std::map FieldBCType_algo_to_enum = { {"pml", FieldBoundaryType::PML}, {"periodic", FieldBoundaryType::Periodic}, {"pec", FieldBoundaryType::PEC}, @@ -222,21 +222,21 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ return algo_to_int[algo]; } -int +FieldBoundaryType GetFieldBCTypeInteger( std::string BCType ){ std::transform(BCType.begin(), BCType.end(), BCType.begin(), ::tolower); - if (FieldBCType_algo_to_int.count(BCType) == 0) { + if (FieldBCType_algo_to_enum.count(BCType) == 0) { std::string error_message = "Invalid string for field/particle BC. : " + BCType + "\nThe valid values are : \n"; - for (const auto &valid_pair : FieldBCType_algo_to_int) { + for (const auto &valid_pair : FieldBCType_algo_to_enum) { if (valid_pair.first != "default"){ error_message += " - " + valid_pair.first + "\n"; } } WARPX_ABORT_WITH_MESSAGE(error_message); } - // return FieldBCType_algo_to_int[BCType]; // This operator cannot be used for a const map - return FieldBCType_algo_to_int.at(BCType); + // return FieldBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map + return FieldBCType_algo_to_enum.at(BCType); } ParticleBoundaryType @@ -257,9 +257,9 @@ GetParticleBCTypeInteger( std::string BCType ){ } std::string -GetFieldBCTypeString( int fb_type ) { +GetFieldBCTypeString( FieldBoundaryType fb_type ) { std::string boundary_name; - for (const auto &valid_pair : FieldBCType_algo_to_int) { + for (const auto &valid_pair : FieldBCType_algo_to_enum) { if ((valid_pair.second == fb_type)&&(valid_pair.first != "default")){ boundary_name = valid_pair.first; break; diff --git a/Source/WarpX.H b/Source/WarpX.H index cdd2447ed5e..9ca754ea1bb 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -75,12 +75,6 @@ #include #include -enum struct PatchType : int -{ - fine, - coarse -}; - enum struct FieldType : int { Efield_aux, @@ -215,12 +209,12 @@ public: * lower domain boundaries * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) */ - static amrex::Vector field_boundary_lo; + static amrex::Vector field_boundary_lo; /** Integers that correspond to boundary condition applied to fields at the * upper domain boundaries * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) */ - static amrex::Vector field_boundary_hi; + static amrex::Vector field_boundary_hi; /** Integers that correspond to boundary condition applied to particles at the * lower domain boundaries * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index f89a61e524b..9a9903f013f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -137,8 +137,8 @@ amrex::IntVect WarpX::shared_tilesize(AMREX_D_DECL(1,1,1)); #endif int WarpX::shared_mem_current_tpb = 128; -amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,0); -amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,0); +amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,FieldBoundaryType::PML); +amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,FieldBoundaryType::PML); amrex::Vector WarpX::particle_boundary_lo(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); amrex::Vector WarpX::particle_boundary_hi(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); From 5640de35aec95dcd2247798d645a7625c3c5807a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 9 Apr 2024 11:08:04 -0700 Subject: [PATCH 055/121] Refactor output of phi on the particles (#4839) * Refactor output of phi on the particles --- Source/Diagnostics/ParticleIO.cpp | 54 +++++++++++++++++++++++++++++ Source/Diagnostics/WarpXOpenPMD.cpp | 27 +-------------- Source/Particles/ParticleIO.H | 45 ++++-------------------- 3 files changed, 62 insertions(+), 64 deletions(-) diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index a8bb9303fe1..5cb18602ed9 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -235,3 +235,57 @@ MultiParticleContainer::WriteHeader (std::ostream& os) const allcontainers.at(i)->WriteHeader(os); } } + +void +storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, + int electrostatic_solver_id, bool is_full_diagnostic ) { + + using PinnedParIter = typename PinnedMemoryParticleContainer::ParIterType; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); + // When this is not a full diagnostic, the particles are not written at the same physical time (i.e. PIC iteration) + // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). + // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. + // To avoid confusion, we raise an error in this case. + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + is_full_diagnostic, + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available with `diag_type = Full`."); + tmp.AddRealComp("phi"); + int const phi_index = tmp.getParticleComps().at("phi"); + auto& warpx = WarpX::GetInstance(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (int lev=0; lev<=warpx.finestLevel(); lev++) { + const amrex::Geometry& geom = warpx.Geom(lev); + auto plo = geom.ProbLoArray(); + auto dxi = geom.InvCellSizeArray(); + amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + + for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { + + auto phi_grid = phi[pti].array(); + const auto getPosition = GetParticlePosition(pti); + amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); + + // Loop over the particles and update their position + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long ip) { + + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); + phi_particle_arr[ip] = phi_value; + } + ); + } + } +} diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 6882b962445..965eb82b31e 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -591,32 +591,7 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // Gather the electrostatic potential (phi) on the macroparticles if ( particle_diags[i].m_plot_phi ) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || - (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), - "Output of the electrostatic potential (phi) on the particles was requested, " - "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); - // Using pinned PC indicates that the particles are not written at the same physical time (i.e. PIC iteration) - // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). - // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. - // To avoid confusion, we raise an error in this case. - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - use_pinned_pc == false, - "Output of the electrostatic potential (phi) on the particles was requested, " - "but this is only available with `diag_type = Full`."); - tmp.AddRealComp("phi"); - int const phi_index = tmp.getParticleComps().at("phi"); - auto& warpx = WarpX::GetInstance(); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (int lev=0; lev<=warpx.finestLevel(); lev++) { - const amrex::Geometry& geom = warpx.Geom(lev); - auto plo = geom.ProbLoArray(); - auto dxi = geom.InvCellSizeArray(); - amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); - storeFieldOnParticles( tmp, plo, dxi, phi, phi_index, lev ); - } + storePhiOnParticles( tmp, WarpX::electrostatic_solver_id, !use_pinned_pc ); } // names of amrex::Real and int particle attributes in SoA data diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index 1abda6ea9af..022ab471362 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -9,6 +9,7 @@ #include "Particles/WarpXParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/PinnedMemoryParticleContainer.H" #include @@ -80,47 +81,15 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer* } } -/** Gathers the field from a MultiFab to the macroparticles - * and stores it into a runtime component of the particle container +/** Gathers phi (electrostatic potential) from a MultiFab to the macroparticles. + * Adds a runtime component of the particle container to store it. * - * @tparam T_ParticleContainer a WarpX particle container or AmrParticleContainer * @param tmp the particle container on which to store the gathered field - * @param plo the coordinates of the low bound of the box, along each dimension - * @param dxi the inverse of the cell size, along each dimension - * @param phi a multifab that contains the field to be gathered - * @param phi_index the index of the runtime component where to store the gathered field - * @param lev the MR level + * @param electrostatic_solver_id the type of electrostatic solver used + * @param is_full_diagnostic whether this diagnostic is a full diagnostic */ -template< typename T_ParticleContainer > void -storeFieldOnParticles ( T_ParticleContainer& tmp, - amrex::GpuArray const& plo, - amrex::GpuArray const& dxi, - amrex::MultiFab const& phi, - int const phi_index, int const lev ) { - - using PinnedParIter = typename T_ParticleContainer::ParIterType; - - for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { - - auto phi_grid = phi[pti].array(); - const auto getPosition = GetParticlePosition(pti); - amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); - - // Loop over the particles and update their position - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long ip) { - - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - int i, j, k; - amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); - amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); - phi_particle_arr[ip] = phi_value; - } - ); - } -} +storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, + int electrostatic_solver_id, bool is_full_diagnostic ); #endif /* PARTICLEIO_H_ */ From a3974ff6b7cf6370c72a4f2c9372058bc1ce803b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 9 Apr 2024 15:29:02 -0700 Subject: [PATCH 056/121] Filter `rho`, when using relativistic electrostatic solver (#4823) * Filter/synchronize rho across levels * Allocate coarse version of `rho` * Remove filtering for now * Remove boundary treatment * Apply boundary and scale volume * Apply suggestions from code review * Update benchmark * Update benchmarks * Increase resolution to mitigate the effects of filtering * Increase resolution in relativistic test * Use filter in openbc test * Update benchmark * Update resolution * Update Examples/Tests/openbc_poisson_solver/inputs_3d * Update checksums --- .../Tests/openbc_poisson_solver/inputs_3d | 5 +- .../inputs_3d | 2 +- .../space_charge_initialization/inputs_3d | 2 +- .../benchmarks_json/BeamBeamCollision.json | 140 +++++++++--------- .../benchmarks_json/ElectrostaticSphere.json | 18 +-- .../ElectrostaticSphereRelNodal.json | 26 ++-- .../NodalElectrostaticSolver.json | 18 +-- .../openbc_poisson_solver.json | 30 ++-- ...ativistic_space_charge_initialization.json | 10 +- .../space_charge_initialization.json | 6 +- .../space_charge_initialization_2d.json | 4 +- Source/FieldSolver/ElectrostaticSolver.cpp | 17 ++- 12 files changed, 145 insertions(+), 133 deletions(-) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/inputs_3d index fd4299f360b..2ac17a06751 100644 --- a/Examples/Tests/openbc_poisson_solver/inputs_3d +++ b/Examples/Tests/openbc_poisson_solver/inputs_3d @@ -8,8 +8,8 @@ amr.n_cell = 128 128 128 amr.max_level = 0 amr.max_grid_size = 128 geometry.dims = 3 -geometry.prob_lo = -5*sigmax -5*sigmay -5*sigmaz -geometry.prob_hi = +5*sigmax +5*sigmay +5*sigmaz +geometry.prob_lo = -4*sigmax -4*sigmay -4*sigmaz +geometry.prob_hi = +4*sigmax +4*sigmay +4*sigmaz boundary.field_lo = open open open boundary.field_hi = open open open warpx.const_dt = 1e-14 @@ -18,7 +18,6 @@ warpx.poisson_solver = fft algo.field_gathering = momentum-conserving algo.particle_shape = 1 - particles.species_names = electron electron.charge = -q_e diff --git a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d b/Examples/Tests/relativistic_space_charge_initialization/inputs_3d index 91cd6000547..c472f88c007 100644 --- a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d +++ b/Examples/Tests/relativistic_space_charge_initialization/inputs_3d @@ -1,5 +1,5 @@ max_step = 1 -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_grid_size = 32 amr.max_level = 0 diff --git a/Examples/Tests/space_charge_initialization/inputs_3d b/Examples/Tests/space_charge_initialization/inputs_3d index a24b67ff02e..c8058fac519 100644 --- a/Examples/Tests/space_charge_initialization/inputs_3d +++ b/Examples/Tests/space_charge_initialization/inputs_3d @@ -1,5 +1,5 @@ max_step = 1 -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_grid_size = 32 amr.max_level = 0 diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json index 693faf43882..799692135ce 100644 --- a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -1,96 +1,96 @@ { "lev=0": { - "Bx": 970573570349.2142, - "By": 970493693614.8823, - "Bz": 18743533.08373785, - "Ex": 2.9100804648817213e+20, - "Ey": 2.910188719414779e+20, - "Ez": 1.309938057448976e+17, - "rho_beam1": 7.969171294602906e+16, - "rho_beam2": 7.969079911431987e+16, - "rho_ele1": 234837475722889.97, - "rho_ele2": 277285508564124.12, - "rho_pos1": 229824415763789.62, - "rho_pos2": 286513388076434.4 + "Bx": 958613893612.4355, + "By": 958606286084.2804, + "Bz": 50291310.73170665, + "Ex": 2.873993867215236e+20, + "Ey": 2.8740263458334176e+20, + "Ez": 1.3469564521662371e+17, + "rho_beam1": 7.96926761425663e+16, + "rho_beam2": 7.969234189119718e+16, + "rho_ele1": 325562788515568.7, + "rho_ele2": 301373974746269.7, + "rho_pos1": 314013278169396.75, + "rho_pos2": 311298336003435.56 }, "beam1": { - "particle_opticalDepthQSR": 104947.58821047074, - "particle_position_x": 0.0015001846430437182, - "particle_position_y": 0.0015002200838688795, - "particle_position_z": 0.004965556992831605, - "particle_momentum_x": 6.207861637949496e-15, - "particle_momentum_y": 6.16591876835476e-15, - "particle_momentum_z": 6.806755306915448e-12, - "particle_weight": 635859587.7915188 + "particle_opticalDepthQSR": 104848.69019384333, + "particle_position_x": 0.0015001641452988207, + "particle_position_y": 0.0015001296473841738, + "particle_position_z": 0.004965480036291212, + "particle_momentum_x": 6.203657034942546e-15, + "particle_momentum_y": 6.161790111190829e-15, + "particle_momentum_z": 6.806194292286189e-12, + "particle_weight": 635868088.3867786 }, "beam2": { - "particle_opticalDepthQSR": 104180.42997257302, - "particle_position_x": 0.0015001132145418016, - "particle_position_y": 0.001500162338878405, - "particle_position_z": 0.004965528365042041, - "particle_momentum_x": 6.201791193788119e-15, - "particle_momentum_y": 6.188946480983165e-15, - "particle_momentum_z": 6.796967564632488e-12, - "particle_weight": 635853166.1373858 + "particle_opticalDepthQSR": 104197.28107371755, + "particle_position_x": 0.0015001452558405398, + "particle_position_y": 0.0015001281739351966, + "particle_position_z": 0.0049656445643994716, + "particle_momentum_x": 6.202758467582172e-15, + "particle_momentum_y": 6.18910011814166e-15, + "particle_momentum_z": 6.7994521022372906e-12, + "particle_weight": 635874794.3085052 }, "ele1": { - "particle_opticalDepthQSR": 385.3959944370225, - "particle_position_x": 5.063867198487091e-06, - "particle_position_y": 5.323267522706671e-06, - "particle_position_z": 1.81974337459613e-05, - "particle_momentum_x": 5.220808698440275e-18, - "particle_momentum_y": 5.2833645967924376e-18, - "particle_momentum_z": 2.4960852396311436e-15, - "particle_weight": 1883777.3071500752 + "particle_opticalDepthQSR": 398.7154177999122, + "particle_position_x": 5.2166787076833645e-06, + "particle_position_y": 5.005755590473258e-06, + "particle_position_z": 1.856829463647771e-05, + "particle_momentum_x": 6.0736878569270085e-18, + "particle_momentum_y": 5.735020185191748e-18, + "particle_momentum_z": 2.827581034346608e-15, + "particle_weight": 2602683.4209351614 }, "ele2": { - "particle_opticalDepthQSR": 391.0032239817431, - "particle_position_x": 5.442400321293304e-06, - "particle_position_y": 5.3621200149906786e-06, - "particle_position_z": 1.716649458876156e-05, - "particle_momentum_x": 4.7056270741663116e-18, - "particle_momentum_y": 4.414438514376292e-18, - "particle_momentum_z": 2.3816287305827113e-15, - "particle_weight": 2255238.1204111334 + "particle_opticalDepthQSR": 328.6975869797729, + "particle_position_x": 4.984003903707884e-06, + "particle_position_y": 4.695016970410262e-06, + "particle_position_z": 1.606918799511055e-05, + "particle_momentum_x": 4.524294388810778e-18, + "particle_momentum_y": 4.193609622515901e-18, + "particle_momentum_z": 2.624217472737641e-15, + "particle_weight": 2432495.8168380223 }, "pho1": { - "particle_opticalDepthBW": 9912.332305353064, - "particle_position_x": 0.0001424857999095925, - "particle_position_y": 0.00014366766674682975, - "particle_position_z": 0.0004764478273708734, + "particle_opticalDepthBW": 10028.214317531058, + "particle_position_x": 0.00014200324200040716, + "particle_position_y": 0.00014310262095706036, + "particle_position_z": 0.00047470309948487784, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61671336.44071695 + "particle_weight": 61455533.15171491 }, "pho2": { - "particle_opticalDepthBW": 10361.142481614075, - "particle_position_x": 0.00014718841339786984, - "particle_position_y": 0.00014538267635727008, - "particle_position_z": 0.00048768591004702896, + "particle_opticalDepthBW": 10261.48950301913, + "particle_position_x": 0.0001465092909391631, + "particle_position_y": 0.00014555115652303745, + "particle_position_z": 0.00048686081947093, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61889405.34553001 + "particle_weight": 61924991.09906147 }, "pos1": { - "particle_opticalDepthQSR": 343.0370605565725, - "particle_position_x": 5.41949101836001e-06, - "particle_position_y": 5.5292865311090835e-06, - "particle_position_z": 1.7063207973991194e-05, - "particle_momentum_x": 4.76704175252458e-18, - "particle_momentum_y": 5.049007427826035e-18, - "particle_momentum_z": 2.533578387785534e-15, - "particle_weight": 1821809.4309160584 + "particle_opticalDepthQSR": 380.4787933889546, + "particle_position_x": 5.59226140958729e-06, + "particle_position_y": 5.199149983019462e-06, + "particle_position_z": 1.7261766049926983e-05, + "particle_momentum_x": 5.182944941041321e-18, + "particle_momentum_y": 4.665394338329992e-18, + "particle_momentum_z": 2.565450485567441e-15, + "particle_weight": 2523696.1743423166 }, "pos2": { - "particle_opticalDepthQSR": 390.0211924195479, - "particle_position_x": 5.053169658257247e-06, - "particle_position_y": 5.039302796539821e-06, - "particle_position_z": 1.8526669235892217e-05, - "particle_momentum_x": 5.755216173987019e-18, - "particle_momentum_y": 5.056618594918483e-18, - "particle_momentum_z": 2.52324147594341e-15, - "particle_weight": 2328558.5523011778 + "particle_opticalDepthQSR": 378.7526306435402, + "particle_position_x": 4.812490588954386e-06, + "particle_position_y": 4.351750384371962e-06, + "particle_position_z": 1.7621416174292307e-05, + "particle_momentum_x": 4.979887438720444e-18, + "particle_momentum_y": 4.8215630209506066e-18, + "particle_momentum_z": 2.193964301475807e-15, + "particle_weight": 2513162.277112609 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json index 4e9bde52a78..45441acf466 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504803536004636, - "Ey": 6.504803536004637, - "Ez": 6.504803536004637, + "Ex": 6.504631859049922, + "Ey": 6.50463185904992, + "Ez": 6.504631859049919, "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 1.0084159184928337e-23, - "particle_momentum_y": 1.008415918492834e-23, - "particle_momentum_z": 1.0084159184928338e-23, - "particle_position_x": 518.418176976446, - "particle_position_y": 518.4181769764461, - "particle_position_z": 518.418176976446, + "particle_momentum_x": 9.882423407879068e-24, + "particle_momentum_y": 9.882423407879068e-24, + "particle_momentum_z": 9.882423407879066e-24, + "particle_position_x": 513.5229059338338, + "particle_position_y": 513.5229059338338, + "particle_position_z": 513.522905933834, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json index 8a79a510352..45441acf466 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json @@ -1,17 +1,17 @@ { - "electron": { - "particle_momentum_x": 1.0084159184928337e-23, - "particle_momentum_y": 1.008415918492834e-23, - "particle_momentum_z": 1.0084159184928338e-23, - "particle_position_x": 518.418176976446, - "particle_position_y": 518.4181769764461, - "particle_position_z": 518.418176976446, - "particle_weight": 6212.501525878906 - }, "lev=0": { - "Ex": 6.504803536004636, - "Ey": 6.504803536004637, - "Ez": 6.504803536004637, + "Ex": 6.504631859049922, + "Ey": 6.50463185904992, + "Ez": 6.504631859049919, "rho": 2.609256800833379e-10 + }, + "electron": { + "particle_momentum_x": 9.882423407879068e-24, + "particle_momentum_y": 9.882423407879068e-24, + "particle_momentum_z": 9.882423407879066e-24, + "particle_position_x": 513.5229059338338, + "particle_position_y": 513.5229059338338, + "particle_position_z": 513.522905933834, + "particle_weight": 6212.501525878906 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json b/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json index 2c60799425a..0692546f6cf 100644 --- a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json +++ b/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json @@ -1,15 +1,15 @@ { "lev=0": { - "Bx": 4966241349161.254, - "By": 4966241349161.246, - "Bz": 6.831134328962481e-11, - "Ex": 1.4888417010888241e+21, - "Ey": 1.4888417010888241e+21, - "Ez": 8138626456.10672, - "g_beam_p": 175096250199.0888, + "Bx": 4964230908893.931, + "By": 4964230908893.93, + "Bz": 7.391430009205359e-11, + "Ex": 1.4882389862594233e+21, + "Ey": 1.4882389862594225e+21, + "Ez": 8130823640.561069, + "g_beam_p": 175096250199.08884, "rho_beam_p": 8.331476384415223e+17, - "vx_beam_p": 3.9150198762467515e-10, - "vy_beam_p": 3.915019770559888e-10, + "vx_beam_p": 3.912093052231238e-10, + "vy_beam_p": 3.9120930706063816e-10, "vz_beam_p": 715792.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json index 50b402b4bab..e4ff1fc68a8 100644 --- a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json +++ b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json @@ -1,20 +1,20 @@ { "lev=0": { - "Bx": 67876594.52390265, - "By": 112770930.05280717, - "Bz": 1.1060167389533003e-13, - "Ex": 3.380787637240387e+16, - "Ey": 2.034889235345363e+16, - "Ez": 2601470.9380206033, - "rho": 5630115646680.221 + "Bx": 100915933.44993827, + "By": 157610622.1855512, + "Bz": 9.717358898362187e-14, + "Ex": 4.7250652706211096e+16, + "Ey": 3.0253948990559976e+16, + "Ez": 3276573.9514776524, + "rho": 10994013582437.193 }, "electron": { - "particle_momentum_x": 6.378073505535043e-19, - "particle_momentum_y": 3.838951101276526e-19, - "particle_momentum_z": 1.1454327682972433e-10, - "particle_position_x": 21.64260864000591, - "particle_position_y": 0.3229614080035443, - "particle_position_z": 12582.912000000004, - "particle_weight": 19972794743.418434 + "particle_momentum_x": 5.701277606050295e-19, + "particle_momentum_y": 3.6504516641520437e-19, + "particle_momentum_z": 1.145432768297242e-10, + "particle_position_x": 17.314086912497864, + "particle_position_y": 0.2583691267187796, + "particle_position_z": 10066.329600000008, + "particle_weight": 19969036501.910976 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json b/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json index d97f8b58053..adb4477378b 100644 --- a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json +++ b/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json @@ -1,11 +1,11 @@ { "lev=0": { - "By": 3.176136185686860e-05, - "Ex": 9.522292819119155e+03, - "Ey": 9.516582886012895e+03, - "Ez": 8.118976924626002, + "By": 0.00025366943932442746, + "Ex": 76051.98704866154, + "Ey": 76009.35086584378, + "Ez": 64.90079149285842, "jx": 0.0, "jy": 0.0, - "jz": 785848.6496458045 + "jz": 6286789.197166436 } } diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization.json b/Regression/Checksum/benchmarks_json/space_charge_initialization.json index e0a157f2a81..07916316a06 100644 --- a/Regression/Checksum/benchmarks_json/space_charge_initialization.json +++ b/Regression/Checksum/benchmarks_json/space_charge_initialization.json @@ -1,8 +1,8 @@ { "lev=0": { - "Ex": 7869.6596181638515, - "Ey": 7863.981043503829, - "Ez": 7869.780361756294, + "Ex": 62838.484914953624, + "Ey": 62796.0798671538, + "Ez": 62839.82091026324, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json b/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json index f37add38914..6ca98880d4a 100644 --- a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json +++ b/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json @@ -1,8 +1,8 @@ { "lev=0": { - "Ex": 7438.125125466449, + "Ex": 29702.801592163793, "Ey": 0.0, - "Ez": 7438.329203373256, + "Ez": 29704.478300339702, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 8f022c308af..d0818959b30 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -164,6 +164,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Allocate fields for charge and potential const int num_levels = max_level + 1; Vector > rho(num_levels); + Vector > rho_coarse(num_levels); // Used in order to interpolate between levels Vector > phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; @@ -174,16 +175,28 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) rho[lev]->setVal(0.); phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); phi[lev]->setVal(0.); + if (lev > 0) { + // For MR levels: allocated the coarsened version of rho + BoxArray cba = nba; + cba.coarsen(refRatio(lev-1)); + rho_coarse[lev] = std::make_unique(cba, DistributionMap(lev), 1, ng); + rho_coarse[lev]->setVal(0.); + } } // Deposit particle charge density (source of Poisson solver) - bool const local = false; + // The options below are identical to those in MultiParticleContainer::DepositCharge + bool const local = true; bool const reset = false; bool const apply_boundary_and_scale_volume = true; + bool const interpolate_across_levels = false; if ( !pc.do_not_deposit) { - pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume); + pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, + interpolate_across_levels); } + SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + // Get the particle beta vector bool const local_average = false; // Average across all MPI ranks std::array beta_pr = pc.meanParticleVelocity(local_average); From e19d66b511f965728b12724441e4a4a6459ad13c Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 9 Apr 2024 19:38:09 -0700 Subject: [PATCH 057/121] Fix out of bounds access to `distance_to_eb` (#4831) * Fix out of bounds access to `distance_to_eb` * Fix clang-tidy error * specify level for redistribute and set `nGrow=1` * Set `local=0` in order to avoid unnecessary communication in the second `Redistribute` call. * Switch `local` back to `1`. * move second redistribute inside EB-only block --- Source/Particles/WarpXParticleContainer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index fd1bc7917ab..ddbfda57c71 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -295,13 +295,16 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, ); } + // Move particles to their appropriate tiles + Redistribute(); + // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + // Call (local) redistribute again to remove particles with invalid ids + Redistribute(0, -1, 0, 1, true); #endif - - Redistribute(); } /* \brief Current Deposition for thread thread_num From 833813d7fefd23487d865c21a6d75a71f0a00283 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 10 Apr 2024 18:10:47 +0200 Subject: [PATCH 058/121] Doc: LUMI (CSC) Update (#4810) * Doc: LUMI (CSC) Update Update the documentation for LUMI (CSC). * LUMI: `-DWarpX_QED_TABLES_GEN_OMP=OFF` Compile issues. --- Docs/source/install/hpc/lumi.rst | 47 +++++-- .../machines/lumi-csc/install_dependencies.sh | 124 +++++++++++++----- .../lumi-csc/{submit.sh => lumi.sbatch} | 2 +- .../lumi-csc/lumi_warpx.profile.example | 27 ++-- 4 files changed, 142 insertions(+), 58 deletions(-) rename Tools/machines/lumi-csc/{submit.sh => lumi.sbatch} (97%) diff --git a/Docs/source/install/hpc/lumi.rst b/Docs/source/install/hpc/lumi.rst index ee65d5b421b..2bae4d21599 100644 --- a/Docs/source/install/hpc/lumi.rst +++ b/Docs/source/install/hpc/lumi.rst @@ -3,7 +3,7 @@ LUMI (CSC) ========== -The `LUMI cluster `_ is located at CSC (Finland). +The `LUMI cluster `__ is located at CSC (Finland). Each node contains 4 AMD MI250X GPUs, each with 2 Graphics Compute Dies (GCDs) for a total of 8 GCDs per node. You can think of the 8 GCDs as 8 separate GPUs, each having 64 GB of high-bandwidth memory (HBM2E). @@ -12,8 +12,11 @@ Introduction If you are new to this system, **please see the following resources**: -* `Lumi user guide `_ -* Batch system: `Slurm `_ +* `Lumi user guide `__ + + * `Project Maintainance `__ and `SSH Key management `__ + * `Quotas and projects `__ +* Batch system: `Slurm `__ * `Data analytics and visualization `__ * `Production directories `__: @@ -48,9 +51,16 @@ Create it now: .. literalinclude:: ../../../../Tools/machines/lumi-csc/lumi_warpx.profile.example :language: bash -Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor -such as ``nano``, ``emacs``, or ``vim`` (all available by default on -LUMI login nodes). +Edit the 2nd line of this script, which sets the ``export proj="project_..."`` variable using a text editor +such as ``nano``, ``emacs``, or ``vim`` (all available by default on LUMI login nodes). +You can find out your project name by running ``lumi-ldap-userinfo`` on LUMI. +For example, if you are member of the project ``project_465000559``, then run ``nano $HOME/lumi_impactx.profile`` and edit line 2 to read: + +.. code-block:: bash + + export proj="project_465000559" + +Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). .. important:: @@ -88,7 +98,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_lumi - cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_lumi/bin/``. @@ -98,7 +108,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lumi_py - cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi_py -j 16 --target pip_install @@ -143,11 +153,26 @@ Running MI250X GPUs (2x64 GB) ^^^^^^^^^^^^^^^^^^^^^ -In non-interactive runs: +The GPU partition on the supercomputer LUMI at CSC has up to `2978 nodes `__, each with 8 Graphics Compute Dies (GCDs). +WarpX runs one MPI rank per Graphics Compute Die. + +For interactive runs, simply use the aliases ``getNode`` or ``runNode ...``. -.. literalinclude:: ../../../../Tools/machines/lumi-csc/submit.sh +The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly). +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` or the concete inputs file. +Copy the executable or point to it via ``EXE`` and adjust the path for the ``INPUTS`` variable accordingly. + +.. literalinclude:: ../../../../Tools/machines/lumi-csc/lumi.sbatch :language: bash - :caption: You can copy this file from ``Tools/machines/lumi-csc/submit.sh``. + :caption: You can copy this file from ``Tools/machines/lumi-csc/lumi.sbatch``. + +To run a simulation, copy the lines above to a file ``lumi.sbatch`` and run + +.. code-block:: bash + + sbatch lumi.sbatch + +to submit the job. .. _post-processing-lumi: diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 0e466f1f57f..814e134bad4 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -14,15 +14,17 @@ set -eu -o pipefail # Check: ###################################################################### # -# Was perlmutter_gpu_warpx.profile sourced and configured correctly? +# Was lumi_warpx.profile sourced and configured correctly? if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lumi_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi # Remove old dependencies ##################################################### # +SRC_DIR="${HOME}/src" SW_DIR="${HOME}/sw/lumi/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} # remove common user mistakes in python, located in .local instead of a venv python3 -m pip uninstall -qq -y pywarpx @@ -33,63 +35,113 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] +if [ -d ${SRC_DIR}/blaspp ] then - cd $HOME/src/blaspp + cd ${SRC_DIR}/blaspp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -rm -rf $HOME/src/blaspp-lumi-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-lumi-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-lumi-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-lumi-gpu-build +rm -rf ${build_dir}/blaspp-lumi-gpu-build +CXX=$(which CC) \ +cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-lumi-gpu-build \ + -Duse_openmp=OFF \ + -Dgpu_backend=hip \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-lumi-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-lumi-gpu-build # LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] +if [ -d ${SRC_DIR}/lapackpp ] then - cd $HOME/src/lapackpp + cd ${SRC_DIR}/lapackpp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -rm -rf $HOME/src/lapackpp-lumi-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-lumi-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-lumi-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-lumi-gpu-build +rm -rf ${build_dir}/lapackpp-lumi-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" \ +cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-lumi-gpu-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-lumi-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-lumi-gpu-build -# c-blosc (I/O compression, for OpenPMD) -if [ -d $HOME/src/c-blosc ] +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ] then - # git repository is already there - : + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc fi -rm -rf $HOME/src/c-blosc-lu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-lu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-lu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-lu-build +rm -rf ${build_dir}/c-blosc-lu-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-lu-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-lu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-lu-build -# ADIOS2 v. 2.8.3 (for OpenPMD) -if [ -d $HOME/src/adios2 ] +# HDF5 (for openPMD) +if [ -d ${SRC_DIR}/hdf5 ] then - # git repository is already there - : + cd ${SRC_DIR}/hdf5 + git fetch --prune + git checkout hdf5-1_14_1-2 + cd - +else + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 +fi +rm -rf ${build_dir}/hdf5-lu-build +cmake -S ${SRC_DIR}/hdf5 \ + -B ${build_dir}/hdf5-lu-build \ + -DBUILD_TESTING=OFF \ + -DHDF5_ENABLE_PARALLEL=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 +cmake --build ${build_dir}/hdf5-lu-build --target install --parallel 10 +rm -rf ${build_dir}/hdf5-lu-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 fi -rm -rf $HOME/src/adios2-lu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-lu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/adios2-2.8.3 -cmake --build $HOME/src/adios2-lu-build --target install -j 16 -rm -rf $HOME/src/adios2-lu-build +rm -rf ${build_dir}/adios2-lu-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-lu-build \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_HDF5=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/adios2-2.8.3 +cmake --build ${build_dir}/adios2-lu-build --target install -j 16 +rm -rf ${build_dir}/adios2-lu-build # Python ###################################################################### @@ -114,9 +166,9 @@ python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt # optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +python3 -m pip install -r ${SRC_DIR}/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 -#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/lumi-csc/submit.sh b/Tools/machines/lumi-csc/lumi.sbatch similarity index 97% rename from Tools/machines/lumi-csc/submit.sh rename to Tools/machines/lumi-csc/lumi.sbatch index d784471acd5..6d4ce1ee37f 100644 --- a/Tools/machines/lumi-csc/submit.sh +++ b/Tools/machines/lumi-csc/lumi.sbatch @@ -20,7 +20,7 @@ date export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # Seen since August 2023 seen on OLCF (not yet seen on LUMI?) -# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors # https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors #export MPICH_SMP_SINGLE_COPY_MODE=NONE #export FI_CXI_RX_MATCH_MODE=software diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 2cb44035ce4..33aff1946c2 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -1,5 +1,5 @@ # please set your project account -#export proj= +#export proj="project_..." # required dependencies module load LUMI/23.09 partition/G @@ -10,23 +10,30 @@ module load buildtools/23.09 module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +SW_DIR="${HOME}/sw/lumi/gpu" +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load Boost/1.82.0-cpeCray-23.09 # optional: for openPMD support -module load cray-hdf5/1.12.2.7 -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble module load cray-python/3.10.10 +if [ -d "${SW_DIR}/venvs/warpx-lumi" ] +then + source ${SW_DIR}/venvs/warpx-lumi/bin/activate +fi + # an alias to request an interactive batch node for one hour # for paralle execution, start on the batch node: srun alias getNode="salloc -A $proj -J warpx -t 01:00:00 -p dev-g -N 1 --ntasks-per-node=8 --gpus-per-task=1 --gpu-bind=closest" @@ -50,4 +57,4 @@ export CXX=$(which amdclang++) export FC=$(which amdflang) export CFLAGS="-I${ROCM_PATH}/include" export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" +export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" From d8fab99e05fd30420eab1a43b5166f45aff11fbc Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 11 Apr 2024 00:14:41 +0200 Subject: [PATCH 059/121] pybind11: v2.12.0 (#4844) Update pybinde11 to v2.12.0: https://github.com/pybind/pybind11/releases/tag/v2.12.0 --- cmake/dependencies/pybind11.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index e65cd206f34..0a7ec260493 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.11.1 CONFIG REQUIRED) + find_package(pybind11 2.12.0 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.11.1" +set(WarpX_pybind11_branch "v2.12.0" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") From 7fca90521b74f546b47ca74c7c5b596d0f38ff05 Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:08:29 -0700 Subject: [PATCH 060/121] Add ohms law hybrid multifab redistribution (#4848) * Initial MF redistribution for load balancing in Hybrid Ohms law solver. * Redistributing temp current multifab since it stores information between steps. * Adding rho_temp multifab to be redistruted. --- Source/Initialization/WarpXInitData.cpp | 6 ------ Source/Parallelization/WarpXRegrid.cpp | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index f0cf82e04a9..5134a3aaa10 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1297,12 +1297,6 @@ void WarpX::CheckKnownIssues() ablastr::warn_manager::WarnPriority::low); } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !load_balance_intervals.isActivated(), - "The hybrid-PIC algorithm involves multifabs that are not yet " - "properly redistributed during load balancing events." - ); - const bool external_particle_field_used = ( mypc->m_B_ext_particle_s != "none" || mypc->m_E_ext_particle_s != "none" ); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 7b0f135f5c8..22979912a84 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -11,6 +11,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" @@ -206,6 +207,11 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(Efield_avg_fp[lev][idim], dm, true ,lev); RemakeMultiFab(Bfield_avg_fp[lev][idim], dm, true ,lev); } + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], dm, false, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim], dm, true, lev); + } #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { RemakeMultiFab(m_edge_lengths[lev][idim], dm, false ,lev); @@ -228,6 +234,11 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // the last step as the initial guess for the next solve RemakeMultiFab(phi_fp[lev], dm, true ,lev); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], dm, false, lev); + } + #ifdef AMREX_USE_EB RemakeMultiFab(m_distance_to_eb[lev], dm, false ,lev); From 2009612b0b308e2d59ac979ee05131b74f9cb78b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 10 Apr 2024 17:12:31 -0700 Subject: [PATCH 061/121] Use function to remove invalid particles without MPI exchange (#4849) * Use function to remove particles * Fix compilation * Make deleteInvalidParticles a separate function * Apply suggestions from code review --- Source/Particles/WarpXParticleContainer.H | 7 +++++++ Source/Particles/WarpXParticleContainer.cpp | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 4b01f09e27e..61a87a12830 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -322,6 +322,13 @@ public: int nattr_int, amrex::Vector> const & attr_int, int uniqueparticles, amrex::Long id=-1); + /** Remove particles with invalid ID + * + * This is a local operation (no MPI communication) and is thus preferable over `ReDistribute` + * (which also removes invalid particles, but involves MPI communications at the same time) + */ + void deleteInvalidParticles (); + virtual void ReadHeader (std::istream& is) = 0; virtual void WriteHeader (std::ostream& os) const = 0; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index ddbfda57c71..142e973d1b2 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -302,11 +302,24 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); - // Call (local) redistribute again to remove particles with invalid ids - Redistribute(0, -1, 0, 1, true); + deleteInvalidParticles(); #endif } +void +WarpXParticleContainer::deleteInvalidParticles () { + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { + ParticleTileType& ptile = ParticlesAt(lev, pti); + removeInvalidParticles( ptile ); + } + } +} + /* \brief Current Deposition for thread thread_num * \param pti Particle iterator * \param wp Array of particle weights From e5b9b5247297dcf1a2765eb680c2f574f75665bb Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 10 Apr 2024 18:13:42 -0700 Subject: [PATCH 062/121] Use `BinaryCollision` class for DSMC collisions (#4744) * use `BinaryCollision` class for DSMC collisions * comment out explicit delete of `ScatteringProcess` constructor * fix first GPU compilation issue * fix RZ issue * avoid copying `binary_collision_functor` * fix merge error * do not keep `ScatteringProcess` instances in `DSMCFunc` * only keep array of `ScatteringProcess::Executor`s * fix clang-tidy errors and reset benchmark * fix default collision type issue * copy cross-section data to `DSMCFunc` * update checksum after fixing issue with cross-section arrays * move logic to set up cross-section interpolators to `DSMCFunc.cpp` file * use internal `struct`s in collision filter functors * clean-up and fixing clang-tidy issues * second clang-tidy fix attempt * Only call redistribute if collision type has product species * fix clang-tidy error * another clang-tidy error fix * Appropriately set `local` parameter in `Redistribute` call. * set `Redistribute` arguments appropriately * check if weight is numerically zero * Apply suggestions from code review Co-authored-by: Remi Lehe * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add docstring for `CollisionPairFilter` function arguments * Use `insert` rather than `push_back` & `rotate` Co-authored-by: Remi Lehe --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../capacitive_discharge/PICMI_inputs_1d.py | 5 +- .../BinaryCollision/BinaryCollision.H | 47 +- .../BinaryCollision/BinaryCollisionUtils.H | 2 + .../BinaryCollision/BinaryCollisionUtils.cpp | 220 +++++----- .../Coulomb/PairWiseCoulombCollisionFunc.H | 70 +-- .../BinaryCollision/DSMC/CMakeLists.txt | 3 +- .../DSMC/CollisionFilterFunc.H | 190 ++------- .../Collision/BinaryCollision/DSMC/DSMC.H | 86 ---- .../Collision/BinaryCollision/DSMC/DSMC.cpp | 300 ------------- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 197 +++++++++ .../BinaryCollision/DSMC/DSMCFunc.cpp | 78 ++++ .../BinaryCollision/DSMC/Make.package | 3 +- .../DSMC/SplitAndScatterFunc.H | 403 +++++++++++------- .../DSMC/SplitAndScatterFunc.cpp | 43 ++ .../NuclearFusion/NuclearFusionFunc.H | 122 +++--- .../Particles/Collision/CollisionHandler.cpp | 16 +- 16 files changed, 890 insertions(+), 895 deletions(-) delete mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H delete mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index d54800debd6..0b59516ccc7 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -183,11 +183,10 @@ def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): self.max_steps = 50 self.diag_steps = 5 self.mcc_subcycling_steps = 2 + self.rng = np.random.default_rng(23094290) else: self.mcc_subcycling_steps = None - - if self.dsmc: - self.rng = np.random.default_rng(23094290) + self.rng = np.random.default_rng() self.ion_density_array = np.zeros(self.nz + 1) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index a9e081d44b4..b36c28db662 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -8,6 +8,7 @@ #define WARPX_PARTICLES_COLLISION_BINARYCOLLISION_H_ #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" @@ -58,14 +59,14 @@ /** * \brief This class performs generic binary collisions. * - * \tparam CollisionFunctorType the type of the specific binary collision functor that acts on a + * \tparam CollisionFunctor the specific binary collision functor that acts on a * single cell - * \tparam CopyTransformFunctorType the type of the second functor used in the case of + * \tparam CopyTransformFunctor the second functor used in the case of * particle creation * */ -template +template class BinaryCollision final : public CollisionBase { @@ -92,18 +93,27 @@ public: WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); } + const CollisionType collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_isSameSpecies = (m_species_names[0] == m_species_names[1]); - m_binary_collision_functor = CollisionFunctorType(collision_name, mypc, m_isSameSpecies); + m_binary_collision_functor = CollisionFunctor(collision_name, mypc, m_isSameSpecies); const amrex::ParmParse pp_collision_name(collision_name); pp_collision_name.queryarr("product_species", m_product_species); + + // if DSMC the colliding species are also product species + // Therefore, we insert the colliding species at the beginning of `m_product_species` + if (collision_type == CollisionType::DSMC) { + m_product_species.insert( m_product_species.begin(), m_species_names.begin(), m_species_names.end() ); + } m_have_product_species = !m_product_species.empty(); - if ((std::is_same::value) & (m_have_product_species)) { + + if ((std::is_same::value) && (m_have_product_species)) { WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + " does not produce species. Thus, `product_species` should not be specified in the input script." ); } - m_copy_transform_functor = CopyTransformFunctorType(collision_name, mypc); + m_copy_transform_functor = CopyTransformFunctor(collision_name, mypc); } ~BinaryCollision () override = default; @@ -111,7 +121,6 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) = delete; BinaryCollision& operator= ( BinaryCollision&& ) = delete; @@ -157,8 +166,8 @@ public: amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, copy_species2.begin(), copy_species2.end(), device_copy_species2.begin()); amrex::Gpu::streamSynchronize(); - auto copy_species1_data = device_copy_species1.data(); - auto copy_species2_data = device_copy_species2.data(); + auto *copy_species1_data = device_copy_species1.data(); + auto *copy_species2_data = device_copy_species2.data(); #else auto *copy_species1_data = copy_species1.data(); auto *copy_species2_data = copy_species2.data(); @@ -190,7 +199,7 @@ public: auto wt = static_cast(amrex::second()); doCollisionsWithinTile( dt, lev, mfi, species1, species2, product_species_vector, - copy_species1_data, copy_species2_data); + copy_species1_data, copy_species2_data); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { @@ -199,6 +208,15 @@ public: amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } + + if (m_have_product_species) { + // The fact that there are product species indicates that particles of + // the colliding species (`species1` and `species2`) may be removed + // (i.e., marked as invalid) in the process of creating new product particles. + // Call (local) redistribute to remove invalid particles + species1.Redistribute(lev, lev, 0, 1, true); + species2.Redistribute(lev, lev, 0, 1, true); + } } } @@ -224,7 +242,7 @@ public: using namespace ParticleUtils; using namespace amrex::literals; - CollisionFunctorType binary_collision_functor = m_binary_collision_functor; + const auto& binary_collision_functor = m_binary_collision_functor.executor(); const bool have_product_species = m_have_product_species; // Store product species data in vectors @@ -522,7 +540,6 @@ public: } ); - // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, @@ -550,9 +567,9 @@ private: bool m_have_product_species; amrex::Vector m_product_species; // functor that performs collisions within a cell - CollisionFunctorType m_binary_collision_functor; + CollisionFunctor m_binary_collision_functor; // functor that creates new particles and initializes their parameters - CopyTransformFunctorType m_copy_transform_functor; + CopyTransformFunctor m_copy_transform_functor; }; diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index f6aa8e6e303..fdc88a275a0 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -19,6 +19,8 @@ enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, DeuteriumHeliumToProtonHeliumFusion, ProtonBoronToAlphasFusion, + DSMC, + PairwiseCoulomb, Undefined }; enum struct NuclearFusionType { diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 812be089462..05756583554 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -17,123 +17,129 @@ namespace BinaryCollisionUtils{ + CollisionType get_collision_type (const std::string& collision_name, + MultiParticleContainer const * const mypc) + { + const amrex::ParmParse pp_collision_name(collision_name); + // For legacy, pairwisecoulomb is the default + std::string type = "pairwisecoulomb"; + pp_collision_name.query("type", type); + if (type == "pairwisecoulomb") { + return CollisionType::PairwiseCoulomb; + } + else if (type == "nuclearfusion") { + const NuclearFusionType fusion_type = get_nuclear_fusion_type(collision_name, mypc); + return nuclear_fusion_type_to_collision_type(fusion_type); + } + else if (type == "dsmc") { + return CollisionType::DSMC; + } + return CollisionType::Undefined; + } + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); - amrex::Vector species_names; - pp_collision_name.getarr("species", species_names); - auto& species1 = mypc->GetParticleContainerFromName(species_names[0]); - auto& species2 = mypc->GetParticleContainerFromName(species_names[1]); - amrex::Vector product_species_name; - pp_collision_name.getarr("product_species", product_species_name); + { + const amrex::ParmParse pp_collision_name(collision_name); + amrex::Vector species_names; + pp_collision_name.getarr("species", species_names); + auto& species1 = mypc->GetParticleContainerFromName(species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(species_names[1]); + amrex::Vector product_species_name; + pp_collision_name.getarr("product_species", product_species_name); - if ((species1.AmIA() && species2.AmIA()) - || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-tritium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) - || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-tritium fusion must be of type neutron and helium4"); - return NuclearFusionType::DeuteriumTritiumToNeutronHelium; - } - else if (species1.AmIA() && species2.AmIA()) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-deuterium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA())){ - return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; - } else if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA())){ - return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; - } else { - WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); - } - } - else if ((species1.AmIA() && species2.AmIA()) + if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-tritium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (product_species1.AmIA() && product_species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-helium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) - || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); - return NuclearFusionType::DeuteriumHeliumToProtonHelium; + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-tritium fusion must be of type neutron and helium4"); + return NuclearFusionType::DeuteriumTritiumToNeutronHelium; + } + else if (species1.AmIA() && species2.AmIA()) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-deuterium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + if ( + (product_species1.AmIA() && product_species2.AmIA()) + ||(product_species1.AmIA() && product_species2.AmIA())){ + return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; + } else if ( + (product_species1.AmIA() && product_species2.AmIA()) + ||(product_species1.AmIA() && product_species2.AmIA())){ + return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; + } else { + WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); } - else if ((species1.AmIA() && species2.AmIA()) + } + else if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-helium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (product_species1.AmIA() && product_species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 1, - "ERROR: Proton-boron must contain exactly one product species"); - auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species.AmIA(), - "ERROR: Product species of proton-boron fusion must be of type helium4"); - return NuclearFusionType::ProtonBoronToAlphas; - } - WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + - species_names[0] + " of type " + species1.getSpeciesTypeName() + - " and species " + species_names[1] + " of type " + - species2.getSpeciesTypeName()); - return NuclearFusionType::Undefined; + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); + return NuclearFusionType::DeuteriumHeliumToProtonHelium; } - - CollisionType get_collision_type (const std::string& collision_name, - MultiParticleContainer const * const mypc) + else if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) { - const amrex::ParmParse pp_collision_name(collision_name); - std::string type; - pp_collision_name.get("type", type); - if (type == "nuclearfusion") { - const NuclearFusionType fusion_type = get_nuclear_fusion_type(collision_name, mypc); - return nuclear_fusion_type_to_collision_type(fusion_type); - } - WARPX_ABORT_WITH_MESSAGE(type + " is not a valid type of collision that creates new particles"); - return CollisionType::Undefined; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 1, + "ERROR: Proton-boron must contain exactly one product species"); + auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species.AmIA(), + "ERROR: Product species of proton-boron fusion must be of type helium4"); + return NuclearFusionType::ProtonBoronToAlphas; } + WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + + species_names[0] + " of type " + species1.getSpeciesTypeName() + + " and species " + species_names[1] + " of type " + + species2.getSpeciesTypeName()); + return NuclearFusionType::Undefined; + } CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) - { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { - return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { - return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { - return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { - return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - } - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - return CollisionType::ProtonBoronToAlphasFusion; - } - WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); - return CollisionType::Undefined; + { + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { + return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { + return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { + return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { + return CollisionType::DeuteriumHeliumToProtonHeliumFusion; + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { + return CollisionType::ProtonBoronToAlphasFusion; } + WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); + return CollisionType::Undefined; + } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index d1a92ee9716..b31fa3922b9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -58,37 +58,41 @@ public: utils::parser::queryWithParser( pp_collision_name, "CoulombLog", CoulombLog); m_CoulombLog = CoulombLog; + + m_exe.m_CoulombLog = m_CoulombLog; + m_exe.m_isSameSpecies = m_isSameSpecies; } - /** - * \brief operator() of the PairWiseCoulombCollisionFunc class. Performs Coulomb collisions - * at the cell level by calling ElasticCollisionPerez. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. - * @param[in] q1,q2 are charges. - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] engine the random engine. - */ - AMREX_GPU_HOST_DEVICE AMREX_INLINE - void operator() ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, - amrex::ParticleReal const q1, amrex::ParticleReal const q2, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const /*cell_start_pair*/, index_type* /*p_mask*/, - index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, - amrex::ParticleReal* /*p_pair_reaction_weight*/, - amrex::RandomEngine const& engine) const + struct Executor { + /** + * \brief Executor of the PairWiseCoulombCollisionFunc class. Performs Coulomb collisions + * at the cell level by calling ElasticCollisionPerez. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. + * @param[in] q1,q2 are charges. + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const q1, amrex::ParticleReal const q2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const /*cell_start_pair*/, index_type* /*p_mask*/, + index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, + amrex::ParticleReal* /*p_pair_reaction_weight*/, + amrex::RandomEngine const& engine) const { using namespace amrex::literals; @@ -99,9 +103,17 @@ public: dt, m_CoulombLog, dV, engine, m_isSameSpecies); } + amrex::ParticleReal m_CoulombLog; + bool m_isSameSpecies; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + private: amrex::ParticleReal m_CoulombLog; bool m_isSameSpecies; + + Executor m_exe; }; #endif // PAIRWISE_COULOMB_COLLISION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt index 3575d53ad29..60da82ee48f 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - DSMC.cpp + DSMCFunc.cpp + SplitAndScatterFunc.cpp ) endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index 64f05650d1f..bc11e5e4a99 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -18,59 +18,69 @@ * \brief This function determines whether a collision occurs for a given * pair of particles. * - * @param[in] u1x,u1y,u1z momenta of the first colliding particle - * @param[in] u2x,u2y,u2z momenta of the second colliding particle - * @param[in] m1,m2 masses - * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] u1x,u1y,u1z momenta of the first colliding particle. + * @param[in] u2x,u2y,u2z momenta of the second colliding particle. + * @param[in] m1,m2 masses. + * @param[in] w1,w2 effective weight of the colliding particles. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. - * @param[in] pair_index is the index of the colliding pair + * @param[in] pair_index is the index of the colliding pair. * @param[out] p_mask is a mask that will be set to a non-zero integer if a * collision occurs. The integer encodes the scattering process. - * @param[out] p_pair_reaction_weight stores the weight of the product particles - * @param[in] process_count number of scattering processes to consider - * @param[in] scattering processes an array of scattering processes included for consideration + * @param[out] p_pair_reaction_weight stores the weight of the product particles. + * @param[in] multiplier factor by which the collision probability is increased to + * account for all other possible binary collision partners. + * @param[in] process_count number of scattering processes to consider. + * @param[in] scattering processes an array of scattering processes included for consideration. * @param[in] engine the random engine. */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, - const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, - const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, - const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, - amrex::ParticleReal w1, amrex::ParticleReal w2, - const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, +void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleReal u1y, + const amrex::ParticleReal u1z, const amrex::ParticleReal u2x, + const amrex::ParticleReal u2y, const amrex::ParticleReal u2z, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const amrex::ParticleReal w1, const amrex::ParticleReal w2, + const amrex::Real dt, const amrex::ParticleReal dV, const int pair_index, index_type* AMREX_RESTRICT p_mask, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - const int& multiplier_ratio, - int const process_count, - ScatteringProcess::Executor* scattering_processes, + const int multiplier, + const int process_count, + const ScatteringProcess::Executor* scattering_processes, const amrex::RandomEngine& engine) { - using namespace amrex::literals; - amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; - const amrex::ParticleReal w_min = amrex::min(w1, w2); - const amrex::ParticleReal w_max = amrex::max(w1, w2); - BinaryCollisionUtils::get_collision_parameters( u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, E_coll, v_coll, lab_to_COM_factor); - // convert E_coll to eV + using namespace amrex::literals; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + // convert E_coll from Joule to eV E_coll /= PhysConst::q_e; - amrex::ParticleReal sigma_tot = 0._prt; + // Evaluate the cross-section for each scattering process to determine + // the total collision probability. + AMREX_ASSERT_WITH_MESSAGE( + (process_count < 4), "Too many scattering processes in DSMC routine." + ); + int coll_type[4] = {0, 0, 0, 0}; + amrex::ParticleReal sigma_sums[4] = {0._prt, 0._prt, 0._prt, 0._prt}; for (int ii = 0; ii < process_count; ii++) { - auto const& scattering_process = *(scattering_processes + ii); - sigma_tot += scattering_process.getCrossSection(E_coll); + auto const& scattering_process = scattering_processes[ii]; + coll_type[ii] = int(scattering_process.m_type); + const amrex::ParticleReal sigma = scattering_process.getCrossSection(E_coll); + sigma_sums[ii] = sigma + ((ii == 0) ? 0._prt : sigma_sums[ii-1]); } + const auto sigma_tot = sigma_sums[process_count-1]; // calculate total collision probability const amrex::ParticleReal exponent = ( - lab_to_COM_factor * multiplier_ratio * w_max - * sigma_tot * v_coll * dt / dV + lab_to_COM_factor * multiplier * w_max * sigma_tot * v_coll * dt / dV ); // Compute actual collision probability that is always between zero and one @@ -84,13 +94,10 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR if (amrex::Random(engine) < probability) { const amrex::ParticleReal random_number = amrex::Random(engine); - amrex::ParticleReal sigma = 0._prt; for (int ii = 0; ii < process_count; ii++) { - auto const& scattering_process = *(scattering_processes + ii); - sigma += scattering_process.getCrossSection(E_coll); - if (random_number <= sigma / sigma_tot) + if (random_number <= sigma_sums[ii] / sigma_tot) { - p_mask[pair_index] = int(scattering_process.m_type); + p_mask[pair_index] = coll_type[ii]; p_pair_reaction_weight[pair_index] = w_min; break; } @@ -102,121 +109,4 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR } } -/** - * \brief Function that determines if a collision occurs and if so, what - * type. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in] ptd_1,ptd_2 contain the particle data of the two species - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] cell_start_pair is the start index of the pairs in that cell. - * @param[out] p_mask is a mask that will be set to a non-zero integer if a - * collision occurs. The integer encodes the scattering process. - * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the - * particles of a given pair. They are only needed here to store information that will be used - * later on when actually creating the product particles. - * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only - * needed here to store information that will be used later on when actually creating the - * product particles. - * @param[in] process_count number of scattering processes to consider - * @param[in] scattering processes an array of scattering processes included for consideration - * @param[in] engine the random engine. - */ -template -AMREX_GPU_HOST_DEVICE AMREX_INLINE -void CollisionFilter ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - PData ptd_1, PData ptd_2, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, - index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - int const process_count, - ScatteringProcess::Executor* scattering_processes, - amrex::RandomEngine const& engine) -{ - - amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; - amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; - amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; - - amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; - amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; - amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; - - // Number of macroparticles of each species - const int NI1 = I1e - I1s; - const int NI2 = I2e - I2s; - const int max_N = amrex::max(NI1,NI2); - - int i1 = I1s; - int i2 = I2s; - int pair_index = cell_start_pair; - - // Because the number of particles of each species is not always equal (NI1 != NI2 - // in general), some macroparticles will be paired with multiple macroparticles of the - // other species and we need to decrease their weight accordingly. - // c1 corresponds to the minimum number of times a particle of species 1 will be paired - // with a particle of species 2. Same for c2. - const int c1 = amrex::max(NI2/NI1,1); - const int c2 = amrex::max(NI1/NI2,1); - -#if (defined WARPX_DIM_RZ) - amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; - amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; -#endif - - for (int k = 0; k < max_N; ++k) - { - // c1k : how many times the current particle of species 1 is paired with a particle - // of species 2. Same for c2k. - const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; - const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; - -#if (defined WARPX_DIM_RZ) - /* In RZ geometry, macroparticles can collide with other macroparticles - * in the same *cylindrical* cell. For this reason, collisions between macroparticles - * are actually not local in space. In this case, the underlying assumption is that - * particles within the same cylindrical cell represent a cylindrically-symmetry - * momentum distribution function. Therefore, here, we temporarily rotate the - * momentum of one of the macroparticles in agreement with this cylindrical symmetry. - * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; - * there is a corresponding assert statement at initialization.) */ - amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; - amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); - u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); -#endif - - CollisionPairFilter( - u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], - u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], - m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, - dt, dV, pair_index, p_mask, p_pair_reaction_weight, - max_N, process_count, scattering_processes, engine); - -#if (defined WARPX_DIM_RZ) - amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); - u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); -#endif - - p_pair_indices_1[pair_index] = I1[i1]; - p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } - ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } - ++pair_index; - } -} - #endif // COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H deleted file mode 100644 index aa656a3d92e..00000000000 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright 2023 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: Roelof Groenewald (TAE Technologies) - * - * License: BSD-3-Clause-LBNL - */ -#ifndef DSMC_H_ -#define DSMC_H_ - -#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" -#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" -#include "Particles/Collision/CollisionBase.H" -#include "Particles/Collision/ScatteringProcess.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleCreation/SmartCopy.H" -#include "Particles/ParticleCreation/SmartUtils.H" -#include "Particles/WarpXParticleContainer.H" -#include "Utils/Parser/ParserUtils.H" -#include "Utils/ParticleUtils.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include -#include - - -/** - * \brief This class performs DSMC (direct simulation Monte Carlo) collisions - * within a cell. Particles are paired up and for each pair a stochastic process - * determines whether a collision occurs. The algorithm is similar to the one - * used for binary Coulomb collisions and the nuclear fusion module. - */ -class DSMC final - : public CollisionBase -{ - // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using index_type = ParticleBins::index_type; - -public: - - /** - * \brief Constructor of the DSMC class - * - * @param[in] collision_name the name of the collision - */ - DSMC (const std::string& collision_name); - - /** Perform the collisions - * - * @param cur_time Current time - * @param dt Time step size - * @param mypc Container of species involved - * - */ - void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; - - /** Perform all binary collisions within a tile - * - * \param[in] lev the mesh-refinement level - * \param[in] mfi iterator for multifab - * \param species_1 first species container - * \param species_2 second species container - * \param copy_species1 SmartCopy for species_1 - * \param copy_species2 SmartCopy for species_2 - * - */ - void doCollisionsWithinTile ( - amrex::Real dt, int lev, amrex::MFIter const& mfi, - WarpXParticleContainer& species_1, - WarpXParticleContainer& species_2, - SmartCopy& copy_species1, - SmartCopy& copy_species2 ); - -private: - amrex::Vector m_scattering_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; -}; - -#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp deleted file mode 100644 index 8f309c3610a..00000000000 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp +++ /dev/null @@ -1,300 +0,0 @@ -/* Copyright 2023 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: Roelof Groenewald (TAE Technologies) - * - * License: BSD-3-Clause-LBNL - */ -#include "CollisionFilterFunc.H" -#include "DSMC.H" -#include "SplitAndScatterFunc.H" - - -DSMC::DSMC (const std::string& collision_name) - : CollisionBase{collision_name} -{ - using namespace amrex::literals; - const amrex::ParmParse pp_collision_name(collision_name); - -#if defined WARPX_DIM_RZ - amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); -#endif - - if(m_species_names.size() != 2) - { - amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); - } - - // query for a list of collision processes - // these could be elastic, excitation, charge_exchange, back, etc. - amrex::Vector scattering_process_names; - pp_collision_name.queryarr("scattering_processes", scattering_process_names); - - // create a vector of ScatteringProcess objects from each scattering - // process name - for (const auto& scattering_process : scattering_process_names) { - const std::string kw_cross_section = scattering_process + "_cross_section"; - std::string cross_section_file; - pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); - - // if the scattering process is excitation or ionization get the - // energy associated with that process - amrex::ParticleReal energy = 0._prt; - if (scattering_process.find("excitation") != std::string::npos || - scattering_process.find("ionization") != std::string::npos) { - const std::string kw_energy = scattering_process + "_energy"; - utils::parser::getWithParser( - pp_collision_name, kw_energy.c_str(), energy); - } - - ScatteringProcess process(scattering_process, cross_section_file, energy); - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, - "Cannot add an unknown scattering process type"); - - m_scattering_processes.push_back(std::move(process)); - } - -#ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - for (auto const& p : m_scattering_processes) { - h_scattering_processes_exe.push_back(p.executor()); - } - m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), - h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); - amrex::Gpu::streamSynchronize(); -#else - for (auto const& p : m_scattering_processes) { - m_scattering_processes_exe.push_back(p.executor()); - } -#endif -} - -void -DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) -{ - WARPX_PROFILE("DSMC::doCollisions()"); - - auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); - auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); - - // SmartCopy objects are created that will facilitate the particle splitting - // operation involved in DSMC collisions between particles with arbitrary - // weights. - const auto copy_factory_species1 = SmartCopyFactory{species1, species1}; - const auto copy_factory_species2 = SmartCopyFactory{species2, species2}; - auto copy_species1 = copy_factory_species1.getSmartCopy(); - auto copy_species2 = copy_factory_species2.getSmartCopy(); - - species1.defineAllParticleTiles(); - species2.defineAllParticleTiles(); - - // Enable tiling - amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } - - // Loop over refinement levels - for (int lev = 0; lev <= species1.finestLevel(); ++lev){ - - amrex::LayoutData* cost = WarpX::getCosts(lev); - - // Loop over all grids/tiles at this level -#ifdef AMREX_USE_OMP - info.SetDynamic(true); -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - } - auto wt = static_cast(amrex::second()); - - doCollisionsWithinTile(dt, lev, mfi, species1, species2, - copy_species1, copy_species2); - - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - wt = static_cast(amrex::second()) - wt; - amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); - } - } - - // Call redistribute to remove particles with negative ids - species1.Redistribute(lev, lev, 0, true, true); - species2.Redistribute(lev, lev, 0, true, true); - } -} - -void -DSMC::doCollisionsWithinTile( - amrex::Real dt, int const lev, amrex::MFIter const& mfi, - WarpXParticleContainer& species_1, - WarpXParticleContainer& species_2, - SmartCopy& copy_species1, - SmartCopy& copy_species2) -{ - using namespace ParticleUtils; - using namespace amrex::literals; - - // get collision processes - auto *scattering_processes = m_scattering_processes_exe.data(); - int const process_count = static_cast(m_scattering_processes_exe.size()); - - // Extract particles in the tile that `mfi` points to - ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); - ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); - - // Find the particles that are in each cell of this tile - ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); - ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); - - // Extract low-level data - int const n_cells = static_cast(bins_1.numBins()); - - // - Species 1 - index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); - index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); - const amrex::ParticleReal m1 = species_1.getMass(); - const auto ptd_1 = ptile_1.getParticleTileData(); - - // - Species 2 - index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); - index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); - const amrex::ParticleReal m2 = species_2.getMass(); - const auto ptd_2 = ptile_2.getParticleTileData(); - - amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); -#if defined WARPX_DIM_1D_Z - auto dV = geom.CellSize(0); -#elif defined WARPX_DIM_XZ - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined WARPX_DIM_RZ - amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box - const auto lo = lbound(cbx); - const auto hi = ubound(cbx); - const int nz = hi.y-lo.y+1; - auto dr = geom.CellSize(0); - auto dz = geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif - - // In the following we set up the "mask" used for creating new particles - // (from splitting events). There is a mask index for every collision - // pair. Below we find the size of the mask based on the greater of the - // number of each species' particle in each cell. - amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); - index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); - - // Compute how many pairs in each cell and store in n_pairs_in_each_cell array - // For different species, the number of pairs in a cell is the number of particles of - // the species that has the most particles in that cell - amrex::ParallelFor( n_cells, - [=] AMREX_GPU_DEVICE (int i_cell) noexcept - { - const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; - const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; - // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) - { - p_n_pairs_in_each_cell[i_cell] = 0; - } - else - { - p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); - } - } - ); - - // Start indices of the pairs in a cell. Will be used for particle creation - amrex::Gpu::DeviceVector pair_offsets(n_cells); - const index_type n_total_pairs = (n_cells == 0) ? 0: - amrex::Scan::ExclusiveSum(n_cells, - p_n_pairs_in_each_cell, pair_offsets.data()); - index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); - - // Now we create the mask. In the DSMC scheme the mask will serve two - // purposes, 1) record whether a given pair should collide and 2) record - // the scattering process that should occur. - amrex::Gpu::DeviceVector mask(n_total_pairs); - index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); - - // Will be filled with the index of the first particle of a given pair - amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); - index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); - // Will be filled with the index of the second particle of a given pair - amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); - index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); - - // How much weight should be given to the produced particle - based on the - // weight of the collision partner which is not split - amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = - pair_reaction_weight.dataPtr(); - - // Loop over cells - amrex::ParallelForRNG( n_cells, - [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept - { - // The particles from species1 that are in the cell `i_cell` are - // given by the `indices_1[cell_start_1:cell_stop_1]` - index_type const cell_start_1 = cell_offsets_1[i_cell]; - index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - // Same for species 2 - index_type const cell_start_2 = cell_offsets_2[i_cell]; - index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; - // Same but for the pairs - index_type const cell_start_pair = p_pair_offsets[i_cell]; - - // ux from species1 can be accessed like this: - // ux_1[ indices_1[i] ], where i is between - // cell_start_1 (inclusive) and cell_start_2 (exclusive) - - // Do not collide if one species is missing in the cell - if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) { return; } - - // shuffle - ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); - ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); -#if defined WARPX_DIM_RZ - const int ri = (i_cell - i_cell%nz) / nz; - auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; -#endif - // Call the function in order to perform collisions - // If there are product species, p_mask, p_pair_indices_1/2, and - // p_pair_reaction_weight are filled here - CollisionFilter( - cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, - indices_1, indices_2, - ptd_1, ptd_2, - m1, m2, dt, dV, - cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, - process_count, scattering_processes, engine ); - } - ); - - const auto num_p_tile1 = ptile_1.numParticles(); - const auto num_p_tile2 = ptile_2.numParticles(); - - // Create the new product particles and define their initial values - // num_added: how many particles of each product species have been created - const int num_added = splitScatteringParticles( - n_total_pairs, - ptile_1, ptile_2, - p_mask, - copy_species1, copy_species2, - m1, m2, - p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight); - - if (num_added > 0) { - setNewParticleIDs(ptile_1, num_p_tile1, num_added); - setNewParticleIDs(ptile_2, num_p_tile2, num_added); - } -} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H new file mode 100644 index 00000000000..1039ee96969 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -0,0 +1,197 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef DSMC_FUNC_H_ +#define DSMC_FUNC_H_ + +#include "CollisionFilterFunc.H" + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMCFunc +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + +public: + + /** + * \brief Constructor of the DSMCFunc class. + */ + DSMCFunc () = default; + + /** + * \brief Constructor of the DSMCFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + * @param[in] isSameSpecies whether the two colliding species are the same + */ + DSMCFunc ( const std::string& collision_name, + MultiParticleContainer const * mypc, + bool isSameSpecies ); + + struct Executor { + /** + * \brief Executor of the DSMCFunc class. Performs DSMC collisions at the cell level. + * Note that this function does not yet create the product particles, but + * instead fills an array p_mask that stores which pairs result in a collision event. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] soa_1,soa_2 contain the struct of array data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given + * pair. It is only needed here to store information that will be used later on when actually + * creating the product particles. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + amrex::RandomEngine const& engine) const + { + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const index_type NI1 = I1e - I1s; + const index_type NI2 = I2e - I2s; + const index_type max_N = amrex::max(NI1,NI2); + + index_type i1 = I1s; + index_type i2 = I2s; + index_type pair_index = cell_start_pair; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 + const index_type c1 = amrex::max(NI2/NI1, index_type(1)); + const index_type c2 = amrex::max(NI1/NI2, index_type(1)); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; +#endif + + for (index_type k = 0; k < max_N; ++k) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const index_type c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const index_type c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) + */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, static_cast(pair_index), p_mask, + p_pair_reaction_weight, static_cast(max_N), + m_process_count, m_scattering_processes_data, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + ++i1; if ( i1 == I1e ) { i1 = I1s; } + ++i2; if ( i2 == I2e ) { i2 = I2s; } + ++pair_index; + } + } + + int m_process_count; + ScatteringProcess::Executor* m_scattering_processes_data; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + + Executor m_exe; +}; + +#endif // DSMC_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp new file mode 100644 index 00000000000..e7a4eb722fc --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -0,0 +1,78 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "DSMCFunc.H" + +/** + * \brief Constructor of the DSMCFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + * @param[in] isSameSpecies whether the two colliding species are the same + */ +DSMCFunc::DSMCFunc ( + const std::string& collision_name, + [[maybe_unused]] MultiParticleContainer const * const mypc, + [[maybe_unused]] const bool isSameSpecies ) +{ + using namespace amrex::literals; + + const amrex::ParmParse pp_collision_name(collision_name); + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + const std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + const std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + + const int process_count = static_cast(m_scattering_processes.size()); + + // Store ScatteringProcess::Executor(s). +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(process_count); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif + + // Link executor to appropriate ScatteringProcess executors + m_exe.m_scattering_processes_data = m_scattering_processes_exe.data(); + m_exe.m_process_count = process_count; +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package index b4cfab89c64..087044e63cd 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -1,3 +1,4 @@ -CEXE_sources += DSMC.cpp +CEXE_sources += DSMCFunc.cpp +CEXE_sources += SplitAndScatterFunc.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index f684b60da78..074226d8227 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -1,4 +1,4 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * @@ -9,173 +9,286 @@ #ifndef SPLIT_AND_SCATTER_FUNC_H_ #define SPLIT_AND_SCATTER_FUNC_H_ +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" -#include "Particles/NamedComponentParticleContainer.H" - -#include +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/ParticleUtils.H" /** - * \brief Function that performs the particle scattering and injection due - * to binary collisions. - * - * \return num_added the number of particles added to each species. + * \brief This class defines an operator to create product particles from DSMC + * collisions and sets the particle properties (position, momentum, weight). */ -template ::value, int> foo = 0> -int splitScatteringParticles ( - const index_type& n_total_pairs, - Tile& ptile1, Tile& ptile2, - const Index* AMREX_RESTRICT mask, - CopyFunc&& copy1, CopyFunc&& copy2, - const amrex::ParticleReal m1, const amrex::ParticleReal m2, - const index_type* AMREX_RESTRICT p_pair_indices_1, - const index_type* AMREX_RESTRICT p_pair_indices_2, - const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +class SplitAndScatterFunc { - using namespace amrex; + // Define shortcuts for frequently-used type names + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - if (n_total_pairs == 0) { - return 0; - } - Gpu::DeviceVector offsets(n_total_pairs); - Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); +public: + /** + * \brief Default constructor of the SplitAndScatterFunc class. + */ + SplitAndScatterFunc () = default; - // The following is used to calculate the appropriate offsets. Note that - // a standard cummulative sum is not appropriate since the mask is also - // used to specify the type of collision and can therefore have values >1 - auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, - [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, - [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, - amrex::Scan::Type::exclusive, amrex::Scan::retSum - ); + /** + * \brief Constructor of the SplitAndScatterFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + */ + SplitAndScatterFunc (const std::string& collision_name, MultiParticleContainer const * mypc); - const auto ptile1_index = ptile1.numParticles(); - const auto ptile2_index = ptile2.numParticles(); - ptile1.resize(ptile1_index + num_added); - ptile2.resize(ptile2_index + num_added); + /** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ + AMREX_INLINE + amrex::Vector operator() ( + const index_type& n_total_pairs, + // Tile& ptile1, Tile& ptile2, + const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + const amrex::Vector& pc_products, + ParticleTileType** AMREX_RESTRICT tile_products, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const amrex::Vector& /*products_mass*/, + const index_type* AMREX_RESTRICT mask, + const amrex::Vector& products_np, + const SmartCopy* AMREX_RESTRICT copy_species1, + const SmartCopy* AMREX_RESTRICT copy_species2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) const + { + using namespace amrex::literals; - const auto ptile1_data = ptile1.getParticleTileData(); - const auto ptile2_data = ptile2.getParticleTileData(); + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } - ParallelForRNG(n_total_pairs, - [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept - { - if (mask[i]) + amrex::Gpu::DeviceVector offsets(n_total_pairs); + index_type* AMREX_RESTRICT offsets_data = offsets.data(); + const index_type* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const total = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (index_type i) -> index_type { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (index_type i, index_type s) { offsets_data[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); + + amrex::Vector num_added_vec(m_num_product_species); + for (int i = 0; i < m_num_product_species; i++) { - // First, make copies of the colliding particles - copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); - copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); - - // Now we adjust the properties of the original and child particles, - // starting with the parent particles - auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; - auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; - uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + // How many particles of product species i are created. + const index_type num_added = total * m_num_products_host[i]; + num_added_vec[i] = static_cast(num_added); + tile_products[i]->resize(products_np[i] + num_added); + } + + // this works for DSMC since the colliding particles are also products + const auto soa_1 = tile_products[0]->getParticleTileData(); + const auto soa_2 = tile_products[1]->getParticleTileData(); + + amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; + + // Create necessary GPU vectors, that will be used in the kernel below + amrex::Vector soa_products; + for (int i = 0; i < m_num_product_species; i++) + { + soa_products.push_back(tile_products[i]->getParticleTileData()); + } +#ifdef AMREX_USE_GPU + amrex::Gpu::DeviceVector device_soa_products(m_num_product_species); + amrex::Gpu::DeviceVector device_products_np(m_num_product_species); + + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, soa_products.begin(), + soa_products.end(), + device_soa_products.begin()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, products_np.begin(), + products_np.end(), + device_products_np.begin()); + + amrex::Gpu::streamSynchronize(); + SoaData_type* AMREX_RESTRICT soa_products_data = device_soa_products.data(); + const index_type* AMREX_RESTRICT products_np_data = device_products_np.data(); +#else + SoaData_type* AMREX_RESTRICT soa_products_data = soa_products.data(); + const index_type* AMREX_RESTRICT products_np_data = products_np.data(); +#endif + + const int* AMREX_RESTRICT p_num_products_device = m_num_products_device.data(); + + amrex::ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept + { + if (mask[i]) { + // for now we ignore the possibility of having actual reaction + // products - only duplicating (splitting) of the colliding + // particles is supported. + + const auto product1_index = products_np_data[0] + + (p_offsets[i]*p_num_products_device[0] + 0); + // Make a copy of the particle from species 1 + copy_species1[0](soa_products_data[0], soa_1, static_cast(p_pair_indices_1[i]), + static_cast(product1_index), engine); + // Set the weight of the new particles to p_pair_reaction_weight[i] + soa_products_data[0].m_rdata[PIdx::w][product1_index] = p_pair_reaction_weight[i]; + + const auto product2_index = products_np_data[1] + + (p_offsets[i]*p_num_products_device[1] + 0); + // Make a copy of the particle from species 2 + copy_species2[1](soa_products_data[1], soa_2, static_cast(p_pair_indices_2[i]), + static_cast(product2_index), engine); + // Set the weight of the new particles to p_pair_reaction_weight[i] + soa_products_data[1].m_rdata[PIdx::w][product2_index] = p_pair_reaction_weight[i]; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights + amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], + -p_pair_reaction_weight[i]); + amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], + -p_pair_reaction_weight[i]); + + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { #if defined(AMREX_USE_OMP) #pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; + idcpu = amrex::ParticleIdCpus::Invalid; #else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); #endif - }; - - // Remove p_pair_reaction_weight[i] from the colliding particles' weights. - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to -1. - Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); - if (w1 <= 0._prt) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); - } + }; - Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); - if (w2 <= 0._prt) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); - } + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to invalid + if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) + { + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + } + if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) + { + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + } - // Set the child particle properties appropriately - ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; - ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; - - auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; - auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; - auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; - auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; - auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; - auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; - - // for simplicity (for now) we assume non-relativistic particles - // and simply calculate the center-of-momentum velocity from the - // rest masses - auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); - auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); - auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); - - // transform to COM frame - ux1 -= uCOM_x; - uy1 -= uCOM_y; - uz1 -= uCOM_z; - ux2 -= uCOM_x; - uy2 -= uCOM_y; - uz2 -= uCOM_z; - - if (mask[i] == int(ScatteringProcessType::ELASTIC)) { - // randomly rotate the velocity vector for the first particle - ParticleUtils::RandomizeVelocity( - ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine - ); - // set the second particles velocity so that the total momentum - // is zero - ux2 = -ux1 * m1 / m2; - uy2 = -uy1 * m1 / m2; - uz2 = -uz1 * m1 / m2; - } else if (mask[i] == int(ScatteringProcessType::BACK)) { - // reverse the velocity vectors of both particles - ux1 *= -1.0_prt; - uy1 *= -1.0_prt; - uz1 *= -1.0_prt; - ux2 *= -1.0_prt; - uy2 *= -1.0_prt; - uz2 *= -1.0_prt; - } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { - if (m1 == m2) { - auto const temp_ux = ux1; - auto const temp_uy = uy1; - auto const temp_uz = uz1; - ux1 = ux2; - uy1 = uy2; - uz1 = uz2; - ux2 = temp_ux; - uy2 = temp_uy; - uz2 = temp_uz; + // Set the child particle properties appropriately + auto& ux1 = soa_products_data[0].m_rdata[PIdx::ux][product1_index]; + auto& uy1 = soa_products_data[0].m_rdata[PIdx::uy][product1_index]; + auto& uz1 = soa_products_data[0].m_rdata[PIdx::uz][product1_index]; + auto& ux2 = soa_products_data[1].m_rdata[PIdx::ux][product2_index]; + auto& uy2 = soa_products_data[1].m_rdata[PIdx::uy][product2_index]; + auto& uz2 = soa_products_data[1].m_rdata[PIdx::uz][product2_index]; + + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (std::abs(m1 - m2) < 1e-28) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + amrex::Abort("Uneven mass charge-exchange not implemented yet."); + } } else { - Abort("Uneven mass charge-exchange not implemented yet."); + amrex::Abort("Unknown scattering process."); } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; } - else { - Abort("Unknown scattering process."); - } - // transform back to labframe - ux1 += uCOM_x; - uy1 += uCOM_y; - uz1 += uCOM_z; - ux2 += uCOM_x; - uy2 += uCOM_y; - uz2 += uCOM_z; + }); + + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + const int start_index = int(products_np[i]); + const int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); } - }); - Gpu::synchronize(); - return static_cast(num_added); -} + amrex::Gpu::synchronize(); + return num_added_vec; + } + +private: + // How many different type of species the collision produces + int m_num_product_species; + // Vectors of size m_num_product_species storing how many particles of a given species are + // produced by a collision event. These vectors are duplicated (one version for host and one + // for device) which is necessary with GPUs but redundant on CPU. + amrex::Gpu::DeviceVector m_num_products_device; + amrex::Gpu::HostVector m_num_products_host; + CollisionType m_collision_type; +}; #endif // SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp new file mode 100644 index 00000000000..de8de9b505d --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp @@ -0,0 +1,43 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "SplitAndScatterFunc.H" + +SplitAndScatterFunc::SplitAndScatterFunc (const std::string& collision_name, + MultiParticleContainer const * const mypc): + m_collision_type{BinaryCollisionUtils::get_collision_type(collision_name, mypc)} +{ + const amrex::ParmParse pp_collision_name(collision_name); + + if (m_collision_type == CollisionType::DSMC) + { + // here we can add logic to deal with cases where products are created, + // for example with impact ionization + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); +#ifndef AMREX_USE_GPU + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); +#endif + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in SplitAndScatterFunc"); + } + +#ifdef AMREX_USE_GPU + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); +#endif +} diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 8f19364baec..b3826114b09 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -77,61 +77,67 @@ public: utils::parser::queryWithParser( pp_collision_name, "fusion_probability_target_value", m_probability_target_value); + + m_exe.m_fusion_multiplier = m_fusion_multiplier; + m_exe.m_probability_threshold = m_probability_threshold; + m_exe.m_probability_target_value = m_probability_target_value; + m_exe.m_fusion_type = m_fusion_type; + m_exe.m_isSameSpecies = m_isSameSpecies; } - /** - * \brief operator() of the NuclearFusionFunc class. Performs nuclear fusions at the cell level - * using the algorithm described in Higginson et al., Journal of Computational Physics 388, - * 439-453 (2019). Note that this function does not yet create the product particles, but - * instead fills an array p_mask that stores which collisions result in a fusion event. - * - * Also note that there are three main differences between this implementation and the - * algorithm described in Higginson's paper: - * - 1) The transformation from the lab frame to the center of mass frame is nonrelativistic - * in Higginson's paper. Here, we implement a relativistic generalization. - * - 2) The behaviour when the estimated fusion probability is greater than one is not - * specified in Higginson's paper. Here, we provide an implementation using two runtime - * dependent parameters (fusion probability threshold and fusion probability target value). See - * documentation for more details. - * - 3) Here, we divide the weight of a particle by the number of times it is paired with other - * particles. This was not explicitly specified in Higginson's paper. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in] soa_1,soa_2 contain the struct of array data of the two species - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] cell_start_pair is the start index of the pairs in that cell. - * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given - * pair. It is only needed here to store information that will be used later on when actually - * creating the product particles. - * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the - * particles of a given pair. They are only needed here to store information that will be used - * later on when actually creating the product particles. - * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only - * needed here to store information that will be used later on when actually creating the - * product particles. - * @param[in] engine the random engine. - */ - AMREX_GPU_HOST_DEVICE AMREX_INLINE - void operator() ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, - amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, - index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - amrex::RandomEngine const& engine) const + struct Executor { + /** + * \brief Executor of the NuclearFusionFunc class. Performs nuclear fusions at the cell level + * using the algorithm described in Higginson et al., Journal of Computational Physics 388, + * 439-453 (2019). Note that this function does not yet create the product particles, but + * instead fills an array p_mask that stores which collisions result in a fusion event. + * + * Also note that there are three main differences between this implementation and the + * algorithm described in Higginson's paper: + * - 1) The transformation from the lab frame to the center of mass frame is nonrelativistic + * in Higginson's paper. Here, we implement a relativistic generalization. + * - 2) The behaviour when the estimated fusion probability is greater than one is not + * specified in Higginson's paper. Here, we provide an implementation using two runtime + * dependent parameters (fusion probability threshold and fusion probability target value). See + * documentation for more details. + * - 3) Here, we divide the weight of a particle by the number of times it is paired with other + * particles. This was not explicitly specified in Higginson's paper. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] soa_1,soa_2 contain the struct of array data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given + * pair. It is only needed here to store information that will be used later on when actually + * creating the product particles. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + amrex::RandomEngine const& engine) const { - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; @@ -213,9 +219,17 @@ public: ++i2; if ( i2 == I2e ) { i2 = I2s; } ++pair_index; } - } + amrex::ParticleReal m_fusion_multiplier; + amrex::ParticleReal m_probability_threshold; + amrex::ParticleReal m_probability_target_value; + NuclearFusionType m_fusion_type; + bool m_isSameSpecies; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + private: // Factor used to increase the number of fusion reaction by decreasing the weight of the // produced particles @@ -230,6 +244,8 @@ private: amrex::ParticleReal m_probability_target_value; NuclearFusionType m_fusion_type; bool m_isSameSpecies; + + Executor m_exe; }; #endif // NUCLEAR_FUSION_FUNC_H_ diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 5abee34c86a..d723a1017d8 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -8,9 +8,10 @@ #include "Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H" #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" -#include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" -#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" +#include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -45,7 +46,8 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) if (type == "pairwisecoulomb") { allcollisions[i] = std::make_unique>( - collision_names[i], mypc); + collision_names[i], mypc + ); } else if (type == "background_mcc") { allcollisions[i] = std::make_unique(collision_names[i]); @@ -54,12 +56,16 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) allcollisions[i] = std::make_unique(collision_names[i]); } else if (type == "dsmc") { - allcollisions[i] = std::make_unique(collision_names[i]); + allcollisions[i] = + std::make_unique>( + collision_names[i], mypc + ); } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( - collision_names[i], mypc); + collision_names[i], mypc + ); } else{ WARPX_ABORT_WITH_MESSAGE("Unknown collision type."); From c8b05b86c277ac93958bdc48169676b3b8bfe170 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 11 Apr 2024 00:06:43 -0700 Subject: [PATCH 063/121] Use local deleteInvalidParticles (instead of Redistribute) in binary collisions (#4851) --- Source/Particles/Collision/BinaryCollision/BinaryCollision.H | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index b36c28db662..807dc827b6b 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -213,9 +213,8 @@ public: // The fact that there are product species indicates that particles of // the colliding species (`species1` and `species2`) may be removed // (i.e., marked as invalid) in the process of creating new product particles. - // Call (local) redistribute to remove invalid particles - species1.Redistribute(lev, lev, 0, 1, true); - species2.Redistribute(lev, lev, 0, 1, true); + species1.deleteInvalidParticles(); + species2.deleteInvalidParticles(); } } } From 8443bce82431a247dea46254b8cb9f41041eef72 Mon Sep 17 00:00:00 2001 From: Ryan Sandberg Date: Fri, 12 Apr 2024 13:49:00 -0700 Subject: [PATCH 064/121] Update optimas/libensemble documentation (#4854) * add openpmd-viewer to conda install instructions * add optimas example to documentation --- Docs/source/install/dependencies.rst | 4 +- Docs/source/usage/workflows.rst | 2 +- Docs/source/usage/workflows/libensemble.rst | 144 ----------- Docs/source/usage/workflows/optimas.rst | 11 + Tools/LibEnsemble/all_machine_specs.py | 26 -- Tools/LibEnsemble/plot_results.py | 128 ---------- Tools/LibEnsemble/read_sim_output.py | 115 --------- Tools/LibEnsemble/requirements.txt | 7 - Tools/LibEnsemble/run_libensemble_on_warpx.py | 224 ------------------ Tools/LibEnsemble/sim/inputs | 215 ----------------- Tools/LibEnsemble/summit_submit_mproc.sh | 56 ----- Tools/LibEnsemble/warpx_simf.py | 110 --------- Tools/LibEnsemble/write_sim_input.py | 118 --------- 13 files changed, 14 insertions(+), 1146 deletions(-) delete mode 100644 Docs/source/usage/workflows/libensemble.rst create mode 100644 Docs/source/usage/workflows/optimas.rst delete mode 100644 Tools/LibEnsemble/all_machine_specs.py delete mode 100755 Tools/LibEnsemble/plot_results.py delete mode 100644 Tools/LibEnsemble/read_sim_output.py delete mode 100644 Tools/LibEnsemble/requirements.txt delete mode 100644 Tools/LibEnsemble/run_libensemble_on_warpx.py delete mode 100644 Tools/LibEnsemble/sim/inputs delete mode 100755 Tools/LibEnsemble/summit_submit_mproc.sh delete mode 100644 Tools/LibEnsemble/warpx_simf.py delete mode 100644 Tools/LibEnsemble/write_sim_input.py diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 4f3544d5c0d..ce9f9dca520 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -80,7 +80,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON @@ -90,7 +90,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv + conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api openpmd-viewer python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv conda activate warpx-cpu-dev # compile WarpX with -DWarpX_MPI=OFF diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 039be199257..5c5329e18b8 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -13,8 +13,8 @@ This section collects typical user workflows and best practices for WarpX. workflows/plot_distribution_mapping workflows/debugging workflows/generate_lookup_tables_with_tools - workflows/libensemble workflows/plot_timestep_duration workflows/psatd_stencil workflows/archiving workflows/ml_dataset_training + workflows/optimas diff --git a/Docs/source/usage/workflows/libensemble.rst b/Docs/source/usage/workflows/libensemble.rst deleted file mode 100644 index af42ae6adf9..00000000000 --- a/Docs/source/usage/workflows/libensemble.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _libensemble: - -Run LibEnsemble on WarpX -======================== - -`LibEnsemble `__ is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. -While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. -If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., LibEnsemble provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (LibEnsemble targets modern HPC platforms like Summit). - -Scripts to run LibEnsemble on WarpX simulations can be found in ``WarpX/Tools/LibEnsemble/``. -This documentation does not aim at giving a training on LibEnsemble, so please refer to the `LibEnsemble documentation `__ for technical details. - -WarpX example problem for LibEnsemble study -------------------------------------------- - -The WarpX example is built on a 2D input file (so that 1 simulation take < 1 min) of a 2-stage laser-wakefield simulation in a boosted frame. -It aims at optimizing emittance preservation in the coupling between two consecutive plasma accelerator stages. -Each stage accelerates an externally-injected electron beam, which charge has been tuned to show a decent amount of Ez field flattening due to longitudinal beam loading. -Each stage has a parabolic transverse profile to guide the laser pulse, and a uniform longitudinal profile with cos-shape ramps and the entrance and at the exit. -A fresh laser pulse is introduced at the entrance of each stage and deleted at the exit. -The beam transverse distribution is matched to the first stage and the beam is injected at the beginning of the plateau of the first stage, so that the emittance is conserved in the first stage. -The two stages are separated by a few-cm gap, and a focusing lens is placed in-between. -Note that this is a very low resolution simulation to show an example, so it is **not** close to numerical convergence. - -In this example, we allow LibEnsemble to tune four **input parameters**: - - - Length of the downramp of the first stage - - - Longitudinal position of the focusing lens (between the two stages) - - - Strength of the focusing lens - - - Length of the downramp of the second stage - -The **output parameter** that LibEnsemble minimizes is the beam emittance at the exit of the second stage, while making sure the charge loss is small. - -The scripts provided can run on a local machine or on the Summit supercomputer at OLCF. -Two options are available: random sampling of parameter space or optimization on the output parameter. -For the latter, we are using the Asynchronously Parallel Optimization Solver for finding Multiple Minima `APOSMM `__ method provided by LibEnsemble. - -Install LibEnsemble -------------------- - -Besides a working WarpX executable, you have to install libEnsemble and its dependencies. - -You can either install all packages via `conda` (recommended), - -.. code-block:: sh - - conda install -c conda-forge libensemble matplotlib numpy scipy yt - -or try to install the same dependencies via `pip` (pick one *or* the other; note our :ref:`installation details on Summit `): - -.. literalinclude:: ../../../../Tools/LibEnsemble/requirements.txt - - -What's in ``Tools/LibEnsemble``? --------------------------------- - -See the `LibEnsemble User Guide `__ for an overview of LibEnsemble concepts. -In a nutshell, a user needs to define - - - A generator function ``gen_f`` that will generate inputs for the simulation, which can be done uniformly, randomly or using an optimizer. - The generator output, i.e., the simulation input, is called ``'x'``. - The generator is provided by LibEnsemble. - When the generator is an optimizer, it takes the simulation output called ``'f'`` as an input. - - - A simulation function ``sim_f`` that will take ``'x'`` as an input and return a single output value ``'f'``. - In our example, ``sim_f`` modifies the WarpX input file depending on ``'x'``, launches a WarpX simulation and reads the simulation output plotfile to extract ``'f'``. - - - An allocator function ``alloc_f`` that will feed available workers with tasks. - This is provided by LibEnsemble. - -The files in ``Tools/LibEnsemble/`` are: - -``run_libensemble_on_warpx.py`` - This is the main LibEnsemble script. - It imports ``gen_f`` and ``alloc_f`` from LibEnsemble, ``sim_f`` from file ``warpx_simf.py`` (see below), defines dictionaries for parameters of each of these objects (``gen_specs`` includes lower and upper bound of each element in the input array ``'x'``, ``alloc_specs`` and ``sim_specs`` respectively) and runs LibEnsemble. - -``warpx_simf.py`` - defines the ``sim_f`` function called ``run_warpx``: - - .. doxygenfunction:: run_warpx - -``sim/inputs`` - WarpX input file. Some of its parameters are modified in ``run_warpx``. - -``write_sim_input.py`` - (util) update one WarpX input file depending on values in ``'x'`` for this run. - - .. doxygenfunction:: write_sim_input - -``read_sim_output.py`` - (util) Read WarpX plotfile and return ``'f'``. - - .. doxygenfunction:: read_sim_output - -``plot_results.py`` - (util) Read LibEnsemble output files ``.npy`` and ``.pickle`` and plot output ``'f'`` (and other output, just for convenience) as a function of input from all simulations. - -``all_machine_specs.py`` - (util) dictionaries of machine-specific parameters. - For convenience, the maximum number of WarpX runs is defined there. - -``summit_submit_mproc.sh`` - Submission script for LibEnsemble+WarpX on Summit. - Make sure to edit this file and add your project ID for allocations. - -Run the example ---------------- - -On Summit or for a local run, LibEnsemble can run with a Random Number Generator (easiest, good as a first try) or with an optimizer (requires python package ``nlopt``). -This is set by the variable ``generator_type`` in ``run_libensemble_on_warpx.py``. -We hereafter assume that all Python modules required are installed and that a WarpX executable is available. - -Run locally -^^^^^^^^^^^ - -Adjust the ``local_specs`` dictionary in ``all_machine_specs.py`` to fix the path to the WarpX executable (and optionally change the number of cores and OpenMP threads), and run - -.. code-block:: sh - - python run_libensemble_on_warpx.py --comms local --nworkers 3 - -This is adapted to a 4-core machine, as it will use: - -- 1 process to run LibEnsemble - -- 1 process (among the 3 workers) to run the generator - -- 2 processes to run 2 concurrent simulations - -Run on Summit at OLCF -^^^^^^^^^^^^^^^^^^^^^ - -- ``cp -r $HOME/warpx/Tools/LibEnsemble/* sim_directory`` -- modify ``run_libensemble_on_warpx.py`` to have ``machine = 'summit'`` -- modify ``all_machine_specs.py`` to put the right path to the WarpX executable -- modify ``summit_submit_mproc.sh`` to set ``LIBE_PLOTS`` to ``false`` and set the project ID -- ``bsub summit_submit_mproc.sh``: - -.. code-block:: sh - - bsub summit_submit_mproc.sh diff --git a/Docs/source/usage/workflows/optimas.rst b/Docs/source/usage/workflows/optimas.rst new file mode 100644 index 00000000000..58e8143b251 --- /dev/null +++ b/Docs/source/usage/workflows/optimas.rst @@ -0,0 +1,11 @@ +.. _optimas: + +Optimizing with Optimas +======================== + +`optimas `__ is an open-source Python library that enables highly scalable parallel optimization, from a typical laptop to exascale HPC systems. +While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. +If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., optimas provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (optimas targets modern HPC platforms like Perlmutter and Frontier). + +A more detailed description of optimas is provided in the `optimas documentation `__. +In particular, the online optimas documentation provides `an example optimization with optimas that runs WarpX simulations `__. diff --git a/Tools/LibEnsemble/all_machine_specs.py b/Tools/LibEnsemble/all_machine_specs.py deleted file mode 100644 index 8dbc2879f61..00000000000 --- a/Tools/LibEnsemble/all_machine_specs.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It contains a dictionary for machine-specific elements. -""" - - -local_specs = { - 'name': 'local', # Machine name - 'cores': 1, # Number of cores per simulation - 'sim_app': os.environ['HOME'] + '/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex', - 'extra_args': '', # extra arguments passed to mpirun/mpiexec at execution - 'OMP_NUM_THREADS': '2', - 'sim_max': 3 # Maximum number of simulations -} - - -summit_specs = { - 'name': 'summit', # Machine name - 'sim_app': os.environ['HOME'] + '/warpx/Bin/main2d.gnu.TPROF.MPI.CUDA.ex', - # extra arguments passed to jsrun at execution - 'extra_args': '-n 1 -a 1 -g 1 -c 1 --bind=packed:1 --smpiargs="-gpu"', - 'OMP_NUM_THREADS': '1', - 'sim_max': 400 # Maximum number of simulations -} diff --git a/Tools/LibEnsemble/plot_results.py b/Tools/LibEnsemble/plot_results.py deleted file mode 100755 index 6c294fe3675..00000000000 --- a/Tools/LibEnsemble/plot_results.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python - -""" -Plotting script for LibEnsemble run on WarpX simulations. -input: A directory which which the script will read the -latest .npy and .pickle files generated by LibEnsemble -""" - -import glob -import os -import pickle -import sys - -import matplotlib.pyplot as plt -import numpy as np - -# 'aposmm' or 'random' -generator_type = 'random' - -print("Reading latest .npy and .pickle files in " + sys.argv[1]) - -# Read latest .npy file, and store the result in results -# * means all if need specific format then *.csv -list_of_files = glob.glob(sys.argv[1] + '*npy') -latest_file = max(list_of_files, key=os.path.getctime) -results = np.load(latest_file) - -# Read latest .pickle file, and store the result in pickle_data -# * means all if need specific format then *.csv -list_of_files = glob.glob(sys.argv[1] + '*pickle') -latest_file = max(list_of_files, key=os.path.getctime) -with open(latest_file, 'rb') as f: - pickle_data = pickle.load(f) -if generator_type == 'aposmm': - nbatches = len(pickle_data[1]['run_order']) - -# Remove un-returned (i.e., not submitted) simulations -results = results[results['returned'] == 1] - -# Re-organize results into a dictionary. Each key is a field in libE_output, -# each value is an array with the value of this field for all WarpX simulations -names = results.dtype.names -results_dict = {} -for i in range(len(names)): - results_dict[names[i]] = np.array([task[i] for task in results]) - -# Sanity checks: exit if a simulation has null charge, as the script -# doesn't handle this (shouldn't be too hard to add though). -if np.min(results_dict['charge'] == 0.0): - sys.exit() -if np.min(results_dict['f'] == 0.0): - sys.exit() - -# Custom: which output parameters are plotted -# as a function of which input parameters -plot_output = ['f', 'emittance', 'charge', 'energy_avg', 'energy_std'] -plot_input = ['ramp_down_1', 'zlens_1', 'adjust_factor', 'ramp_down_2'] - -# For convenience, dictionary with pyplot.ylim -# tuples for all elements in plot_output -d_ylim = { - 'f': (3.e-7, 5.e-3), - 'emittance': (3.e-7, 5.e-3), - 'charge': (1.e-7, 1.e-5), - 'energy_avg': (5.e1, 5.e2), - 'energy_std': (1.e-2, 1.e-1) -} - -# For convenience, dictionary with units for all elements in plot_output -d_yunits = { - 'f': " (a.u.)", - 'emittance': " (m rad)", - 'charge': " (pC/m)", - 'energy_avg': " (MeV)", - 'energy_std': " (%)" -} - -# Print input and output parameters for the optimal run -print("Best run:") -ind_best = np.nanargmin(results_dict['f']) -for key in plot_input + plot_output: - print(key, results_dict[key][ind_best]) -print("charge_f/charge_i ", - results[ind_best]['charge']/np.nanmax(results_dict['charge'])) - -# And now plotting begins -plt.figure(figsize=(16, 10)) -plt.style.use('dark_background') -# Loop over all input parameters in plot_input -for icount, iname in enumerate(plot_input): - # Loop over all output parameters in plot_output - for count, name in enumerate(plot_output): - plt.subplot(len(plot_input), len(plot_output), - len(plot_output)*icount + count+1) - if generator_type == 'aposmm': - # Plot 1 point per run. Runs that are part of the same local - # minimum search are batched together and plotted with the - # same color. - for batch in pickle_data[1]['run_order']: - my_sims = np.isin(results_dict['sim_id'], - pickle_data[1]['run_order'][batch]) - plt.scatter(np.squeeze(results_dict[iname][my_sims]), - np.squeeze(results_dict[name][my_sims]), - s=2) - elif generator_type == 'random': - # Plot 1 point per run. Colorbar stands for sim ID - # (larger value = later run). - plt.scatter(np.squeeze(results_dict[iname]), - np.squeeze(results_dict[name]), - c=results_dict['sim_id'], s=2, cmap='viridis') - cbar = plt.colorbar() - cbar.ax.set_ylabel('start time (arb. units)') - else: - print("generator_type should be either random or aposmm") - sys.exit() - plt.yscale('log') - plt.xlim(np.min(results_dict[iname]), np.max(results_dict[iname])) - plt.ylim(d_ylim[name]) - plt.grid() - plt.title(name) - plt.xlabel(iname) - plt.ylabel(name + d_yunits[name]) - -plt.tight_layout() - -# Save figure -plt.savefig('results.pdf', bbox_inches='tight') -plt.savefig('results.png', bbox_inches='tight') diff --git a/Tools/LibEnsemble/read_sim_output.py b/Tools/LibEnsemble/read_sim_output.py deleted file mode 100644 index 5eabf18030a..00000000000 --- a/Tools/LibEnsemble/read_sim_output.py +++ /dev/null @@ -1,115 +0,0 @@ -import glob -import os -import shutil - -import numpy as np -import scipy.constants as scc -import yt - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It reads the output plotfiles of a simulation and returns beam -quantity relevant for the LibEnsemble optimizations ('f') as well as other -beam quantities for convenience. -""" - -yt.funcs.mylog.setLevel(50) - -def slice_emittance(x, xp, g, b, w): - xpgb = xp * g * b - xav = np.average(x, weights = w) - xpav = np.average(xpgb, weights = w) - xstd2 = np.average((x - xav)**2, weights = w) - xpstd2 = np.average((xpgb - xpav)**2, weights = w) - xp2 = (np.average((x - xav) * (xpgb-xpav), weights = w))**2 - em = np.sqrt(xstd2 * xpstd2 - xp2) - return em - -def _beam_properties(filepath): - """ - Reads plotfile filepath and compute and return beam parameters - - Parameters - ---------- - filepath : path to plotfile to read - """ - - # Read beam quantities from plotfile - ds = yt.load(filepath) - - if ('beam' in [i[0] for i in ds.field_list]): - ad = ds.all_data() - w = ad['beam', 'particle_weight'].v - if (w.shape[0] <= 200): - print('Insufficient particles in ',filepath) - return 0.0, 0.0, 0.0, 0.0 - else: - x = ad['beam', 'particle_position_x'].v - z = ad['beam', 'particle_position_y'].v - ux = ad['beam', 'particle_momentum_x'].v/scc.m_e/scc.c - uy = ad['beam', 'particle_momentum_y'].v/scc.m_e/scc.c - uz = ad['beam', 'particle_momentum_z'].v/scc.m_e/scc.c - - # Compute beam parameters - # Defined like that, the beam charge is > 0. - charge = np.sum(w) * scc.e - gamma = np.sqrt(1. + ux**2 + uy**2 + uz**2) - beta = np.sqrt(1.0 - 1.0 / gamma**2) - energy_MeV = scc.physical_constants["electron mass energy equivalent in MeV"][0] * (gamma - 1.) - energy_avg = np.average(energy_MeV, weights = w) - energy_std = np.sqrt(np.average((energy_MeV - energy_avg)**2, weights = w)) / energy_avg * 100 - nslices = 20 - zslices = np.linspace(np.min(z), np.max(z), nslices+1) - exlist = np.zeros(nslices) - for slicei in range(nslices): - cond = [(z > zslices[slicei]) & (z < zslices[slicei+1])][0] - wslice = w[cond] - if (wslice.shape[0] > 10): - xslice = x[cond] - xpslice = ux[cond]/uz[cond] - gslice = gamma[cond] - bslice = beta[cond] - exlist[slicei] = slice_emittance(xslice, xpslice, gslice, bslice, wslice) - emittance = np.mean(exlist) - return charge, energy_avg, energy_std, emittance - else: - print('No beam particles in ',filepath) - return 0.0, 0.0, 0.0, 0.0 - -def read_sim_output(workdir): - """ - Return optimizing quantity 'f' and other parameters for convenience. - - Parameters - ---------- - workdir : Path to directory where the simulation ran. - """ - # Get beam properties at the beginning of the run - file_list = glob.glob('diags/plotfiles/plt?????') - if (len(file_list) <2): - print(workdir,' did not have final plotfile') - return np.array([np.nan, np.nan, np.nan, np.nan, np.nan]) - file_list.sort() - datafile = file_list[0] - filepath = os.path.join(workdir, datafile) - charge_i, _, _, emittance_i = _beam_properties(filepath) - - # Get beam properties at the end of the run - datafile = file_list[-1] - filepath = os.path.join(workdir, datafile) - charge_f, energy_avg, energy_std, emittance_f = _beam_properties(filepath) - - if (charge_f > 0.0): - # delete simulation results, just to have smaller data - shutil.rmtree('diags') - - # Build a quantity to minimize (f) that encompasses - # emittance AND charge loss 1% charge loss has the - # same impact as doubling the initial emittance. - # we minimize f! - f = emittance_f + emittance_i*(1.-charge_f/charge_i)*100 - warpx_out = np.array([f, energy_std, energy_avg, charge_f, emittance_f]) - - return warpx_out - else: - return np.array([np.nan, np.nan, np.nan, np.nan, np.nan]) diff --git a/Tools/LibEnsemble/requirements.txt b/Tools/LibEnsemble/requirements.txt deleted file mode 100644 index 88217255e85..00000000000 --- a/Tools/LibEnsemble/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -libensemble -matplotlib -nlopt -numpy -pytest -scipy -yt diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py deleted file mode 100644 index d825288dc56..00000000000 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It is the entry point script that runs LibEnsemble. Libensemble -then launches WarpX simulations. - -Execute locally via the following command: - python run_libensemble_on_warpx.py --comms local --nworkers 3 -On summit, use the submission script: - bsub summit_submit_mproc.sh - -The number of concurrent evaluations of the objective function will be -nworkers=1 as one worker is for the persistent gen_f. -""" - -# Either 'random' or 'aposmm' -generator_type = 'aposmm' -# Either 'local' or 'summit' -machine = 'local' - -import sys - -import numpy as np - -# Import libEnsemble modules -from libensemble.libE import libE -from warpx_simf import run_warpx # Sim function from current directory - -if generator_type == 'random': - from libensemble.alloc_funcs.give_sim_work_first import ( - give_sim_work_first as alloc_f, - ) - from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -elif generator_type == 'aposmm': - import libensemble.gen_funcs - libensemble.gen_funcs.rc.aposmm_optimizers = 'nlopt' - from libensemble.alloc_funcs.persistent_aposmm_alloc import ( - persistent_aposmm_alloc as alloc_f, - ) - from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f -else: - print("you shouldn' hit that") - sys.exit() - -import all_machine_specs -from libensemble import libE_logger -from libensemble.executors.mpi_executor import MPIExecutor -from libensemble.tools import add_unique_random_streams, parse_args, save_libE_output - -# Import machine-specific run parameters -if machine == 'local': - machine_specs = all_machine_specs.local_specs -elif machine == 'summit': - machine_specs = all_machine_specs.summit_specs -else: - print("you shouldn' hit that") - sys.exit() - -libE_logger.set_level('INFO') - -nworkers, is_master, libE_specs, _ = parse_args() - -# Set to full path of warp executable -sim_app = machine_specs['sim_app'] - -# Problem dimension. This is the number of input parameters exposed, -# that LibEnsemble will vary in order to minimize a single output parameter. -n = 4 - -exctr = MPIExecutor(central_mode=True) -exctr.register_calc(full_path=sim_app, calc_type='sim') - -# State the objective function, its arguments, output, and necessary parameters -# (and their sizes). Here, the 'user' field is for the user's (in this case, -# the simulation) convenience. Feel free to use it to pass number of nodes, -# number of ranks per note, time limit per simulation etc. -sim_specs = { - # Function whose output is being minimized. The parallel WarpX run is - # launched from run_WarpX. - 'sim_f': run_warpx, - # Name of input for sim_f, that LibEnsemble is allowed to modify. - # May be a 1D array. - 'in': ['x'], - 'out': [ - # f is the single float output that LibEnsemble minimizes. - ('f', float), - # All parameters below are not used for calculation, - # just output for convenience. - # Final relative energy spread. - ('energy_std', float, (1,)), - # Final average energy, in MeV. - ('energy_avg', float, (1,)), - # Final beam charge. - ('charge', float, (1,)), - # Final beam emittance. - ('emittance', float, (1,)), - # input parameter: length of first downramp. - ('ramp_down_1', float, (1,)), - # input parameter: Length of second downramp. - ('ramp_down_2', float, (1,)), - # input parameter: position of the focusing lens. - ('zlens_1', float, (1,)), - # Relative strength of the lens (1. is from - # back-of-the-envelope calculation) - ('adjust_factor', float, (1,)), - ], - 'user': { - # name of input file - 'input_filename': 'inputs', - # Run timeouts after 3 mins - 'sim_kill_minutes': 3, - # machine-specific parameters - 'machine_specs': machine_specs - } -} - -# State the generating function, its arguments, output, -# and necessary parameters. -if generator_type == 'random': - # Here, the 'user' field is for the user's (in this case, - # the RNG) convenience. - gen_specs = { - # Generator function. Will randomly generate new sim inputs 'x'. - 'gen_f': gen_f, - # Generator input. This is a RNG, no need for inputs. - 'in': [], - 'out': [ - # parameters to input into the simulation. - ('x', float, (n,)) - ], - 'user': { - # Total max number of sims running concurrently. - 'gen_batch_size': nworkers, - # Lower bound for the n parameters. - 'lb': np.array([2.e-3, 2.e-3, 0.005, .1]), - # Upper bound for the n parameters. - 'ub': np.array([2.e-2, 2.e-2, 0.028, 3.]), - } - } - - alloc_specs = { - # Allocator function, decides what a worker should do. - # We use a LibEnsemble allocator. - 'alloc_f': alloc_f, - 'out': [ - ('allocated', bool) - ], - 'user': { - # If true wait for all sims to process before generate more - 'batch_mode': True, - # Only one active generator at a time - 'num_active_gens': 1 - } - } - -elif generator_type == 'aposmm': - # Here, the 'user' field is for the user's (in this case, - # the optimizer) convenience. - gen_specs = { - # Generator function. Will randomly generate new sim inputs 'x'. - 'gen_f': gen_f, - 'in': [], - 'out': [ - # parameters to input into the simulation. - ('x', float, (n,)), - # x scaled to a unique cube. - ('x_on_cube', float, (n,)), - # unique ID of simulation. - ('sim_id', int), - # Whether this point is a local minimum. - ('local_min', bool), - # whether the point is from a local optimization run - # or a random sample point. - ('local_pt', bool) - ], - 'user': { - # Number of sims for initial random sampling. - # Optimizer starts afterwards. - 'initial_sample_size': max(nworkers-1, 1), - # APOSMM/NLOPT optimization method - 'localopt_method': 'LN_BOBYQA', - 'num_pts_first_pass': nworkers, - # Relative tolerance of inputs - 'xtol_rel': 1e-3, - # Absolute tolerance of output 'f'. Determines when - # local optimization stops. - 'ftol_abs': 3e-8, - # Lower bound for the n input parameters. - 'lb': np.array([2.e-3, 2.e-3, 0.005, .1]), - # Upper bound for the n input parameters. - 'ub': np.array([2.e-2, 2.e-2, 0.028, 3.]), - } - } - - alloc_specs = { - # Allocator function, decides what a worker should do. - # We use a LibEnsemble allocator. - 'alloc_f': alloc_f, - 'out': [('given_back', bool)], - 'user': {}} - -else: - print("you shouldn' hit that") - sys.exit() - -# Save H to file every N simulation evaluations -libE_specs['save_every_k_sims'] = 100 -# Sim directory to be copied for each worker -libE_specs['sim_input_dir'] = 'sim' - -sim_max = machine_specs['sim_max'] # Maximum number of simulations -exit_criteria = {'sim_max': sim_max} # Exit after running sim_max simulations - -# Create a different random number stream for each worker and the manager -persis_info = add_unique_random_streams({}, nworkers + 1) - -# Run LibEnsemble, and store results in history array H -H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs) - -# Save results to numpy file -if is_master: - save_libE_output(H, persis_info, __file__, nworkers) diff --git a/Tools/LibEnsemble/sim/inputs b/Tools/LibEnsemble/sim/inputs deleted file mode 100644 index 03e9968e5b0..00000000000 --- a/Tools/LibEnsemble/sim/inputs +++ /dev/null @@ -1,215 +0,0 @@ -################################# -########## MESH PATCH ########### -################################# -amr.max_level = 0 - -################################# -######### BOX PARAMETERS ######## -################################# -# warpx.zmax_plasma_to_compute_max_step = 0.33 -warpx.zmax_plasma_to_compute_max_step = -1. -# max_step = 3150 - -amr.n_cell = 32 2048 # 64 3072 - -amr.max_grid_size = 4096 -amr.blocking_factor = 16 - -geometry.dims = 2 -geometry.is_periodic = 0 0 -# physical domain when running in the lab frame -geometry.prob_lo = -0.00024190484157981564 -0.00016126989438654374 -geometry.prob_hi = 0.00024190484157981564 0.0 - -################################# -############ NUMERICS ########### -################################# -warpx.verbose = 1 -algo.current_deposition = esirkepov -algo.charge_deposition = standard -#algo.field_gathering = standard -algo.field_gathering = momentum-conserving -algo.particle_pusher = vay -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 -algo.particle_shape = 3 -warpx.use_filter = 1 -warpx.filter_npass_each_dir = 1 4 -warpx.cfl = .9999 -warpx.do_pml = 0 -warpx.do_dynamic_scheduling = 1 -# Moving window -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # in units of the speed of light - -################################# -####### BOOST PARAMETERS ######## -################################# -warpx.gamma_boost = 30.0 -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = electrons ions electrons2 ions2 beam - -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = beam - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 1 1 -electrons.momentum_distribution_type = "at_rest" -electrons.xmin = -150.e-6 -electrons.xmax = 150.e-6 -electrons.ymin = -150.e-6 -electrons.ymax = 150.e-6 -electrons.zmin = 0.0 -electrons.zmax = 0.32 -electrons.profile = "predefined" -electrons.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = NUniformPerCell -ions.num_particles_per_cell_each_dim = 1 1 -ions.momentum_distribution_type = "at_rest" -ions.xmin = -150.e-6 -ions.xmax = 150.e-6 -ions.ymin = -150.e-6 -ions.ymax = 150.e-6 -ions.zmin = 0.0 -ions.zmax = 0.32 -ions.profile = "predefined" -ions.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 -ions.do_continuous_injection = 1 - -electrons2.charge = -q_e -electrons2.mass = m_e -electrons2.injection_style = NUniformPerCell -electrons2.num_particles_per_cell_each_dim = 1 1 1 -electrons2.momentum_distribution_type = "at_rest" -electrons2.xmin = -150.e-6 -electrons2.xmax = 150.e-6 -electrons2.ymin = -150.e-6 -electrons2.ymax = 150.e-6 -electrons2.zmin = 0.3485 -electrons2.zmax = 0.6685 -electrons2.profile = "predefined" -electrons2.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons2.predefined_profile_params = 0.3485 .02 .297 .003 40.e-6 1.7e23 -electrons2.do_continuous_injection = 1 - -ions2.charge = q_e -ions2.mass = m_p -ions2.injection_style = NUniformPerCell -ions2.num_particles_per_cell_each_dim = 1 1 1 -ions2.momentum_distribution_type = "at_rest" -ions2.xmin = -150.e-6 -ions2.xmax = 150.e-6 -ions2.ymin = -150.e-6 -ions2.ymax = 150.e-6 -ions2.zmin = 0.3485 -ions2.zmax = 0.6685 -ions2.profile = "predefined" -ions2.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions2.predefined_profile_params = 0.3485 .02 .297 .003 40.e-6 1.7e23 -ions2.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = 6.e-7 -beam.y_rms = 6.e-7 -beam.z_rms = 3.e-6 -beam.x_cut = 3. -beam.y_cut = 3. -beam.z_cut = 2. -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -98.e-6 -beam.npart = 100000 -beam.q_tot = -0.9e-12 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0. -beam.uy_m = 0. -beam.uz_m = 1956.9469069265976 -beam.ux_th = 0.4166666666666667 -beam.uy_th = 0. -beam.uz_th = 39.138943248532286 -beam.zinject_plane = 0.02 -beam.rigid_advance = true - -################################# -######### Lens Mirror ########### -################################# - -my_constants.ga = 13330 -my_constants.zlen = 0.3375 -my_constants.dlen = 0.019 -my_constants.wlen = 0.002 -my_constants.mcce = 510999. -my_constants.gab = 30.0 -my_constants.vb = 299625860.344456 -my_constants.adjust_factor = 1. - -particles.E_ext_particle_init_style = parse_E_ext_particle_function -particles.Ex_external_particle_function(x,y,z,t) = "( adjust_factor * gab * 2 * mcce * ga * x / (wlen * dlen) ) * ((gab*(z+vb*t))>=zlen) * ((gab*(z+vb*t))<=(zlen+wlen))" -particles.Ey_external_particle_function(x,y,z,t) = "0." -particles.Ez_external_particle_function(x,y,z,t) = "0." - -particles.B_ext_particle_init_style = parse_B_ext_particle_function -particles.Bx_external_particle_function(x,y,z,t) = "0." -particles.By_external_particle_function(x,y,z,t) = "( - adjust_factor * vb * gab * 2 * mcce * ga * x / (wlen * dlen * clight * clight) ) * ((gab*(z+vb*t))>=zlen) * ((gab*(z+vb*t))<=(zlen+wlen))" -particles.Bz_external_particle_function(x,y,z,t) = "0." - -warpx.num_mirrors=2 -warpx.mirror_z = 0.321 0.6695 -warpx.mirror_z_width = 8.0e-6 8.0e-6 -warpx.mirror_z_npoints = 4 4 - -################################# -############# LASER ############# -################################# -lasers.names = laser1 laser2 - -laser1.profile = Gaussian -laser1.position = 0. 0. -1.e-9 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 0. 1. 0. # The main polarization vector -laser1.e_max = 6.82274e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 50.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 7.33841e-14 # The duration of the laser (in seconds) -laser1.profile_t_peak = 1.46764864e-13 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 0.00875 # Focal distance from the antenna (in meters) -laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) - -laser2.profile = Gaussian -laser2.position = 0. 0. 0.348499999 # This point is on the laser plane -laser2.direction = 0. 0. 1. # The plane normal direction -laser2.polarization = 0. 1. 0. # The main polarization vector -laser2.e_max = 6.82274e12 # Maximum amplitude of the laser field (in V/m) -laser2.profile_waist = 50.e-6 # The waist of the laser (in meters) -laser2.profile_duration = 7.33841e-14 # The duration of the laser (in seconds) -laser2.profile_t_peak = 1.1626176366295598e-09 # The time at which the laser reaches its peak (in seconds) -laser2.profile_focal_distance = 0.00875 # Focal distance from the antenna (in meters) -laser2.wavelength = 0.8e-6 # The wavelength of the laser (in meters) -laser2.do_continuous_injection = 1 # Lasers are initialized outside of initial box - -diagnostics.diags_names = diag -diag.file_prefix = diags/plotfiles/plt -diag.diag_type = Full -diag.intervals = 10000 -diag.species = electrons electrons2 beam -diag.electrons.variables = w -diag.electrons2.variables = w -diag.beam.variables = w ux uy uz diff --git a/Tools/LibEnsemble/summit_submit_mproc.sh b/Tools/LibEnsemble/summit_submit_mproc.sh deleted file mode 100755 index d83c57b0e1e..00000000000 --- a/Tools/LibEnsemble/summit_submit_mproc.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -x -#BSUB -P -#BSUB -J libe_mproc -#BSUB -W 00:30 -#BSUB -nnodes 4 -#BSUB -alloc_flags "smt1" - -# Script to run libEnsemble using multiprocessing on launch nodes. -# Assumes Conda environment is set up. - -# To be run with central job management -# - Manager and workers run on launch node. -# - Workers submit tasks to the nodes in the job available. - -# Name of calling script- -export EXE=run_libensemble_on_warpx.py - -# Communication Method -export COMMS="--comms local" - -# Number of workers. -export NWORKERS="--nworkers 24" - -# Wallclock for libE. Slightly smaller than job wallclock -#export LIBE_WALLCLOCK=15 # Optional if pass to script - -# Name of Conda environment -export CONDA_ENV_NAME=libensemble - -export LIBE_PLOTS=true # Require plot scripts in $PLOT_DIR (see at end) -export PLOT_DIR=.. - -# Need these if not already loaded - -# module load python -# module load gcc -# module load cuda - -module unload xalt - -# Activate conda environment -export PYTHONNOUSERSITE=1 -. activate $CONDA_ENV_NAME - -# hash -d python # Check pick up python in conda env -hash -r # Check no commands hashed (pip/python...) - -# Launch libE. -#python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 -python $EXE $COMMS $NWORKERS > out.txt 2>&1 - -if [[ $LIBE_PLOTS = "true" ]]; then - python $PLOT_DIR/plot_libe_calcs_util_v_time.py - python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libe_histogram.py -fi diff --git a/Tools/LibEnsemble/warpx_simf.py b/Tools/LibEnsemble/warpx_simf.py deleted file mode 100644 index f310dd0f95e..00000000000 --- a/Tools/LibEnsemble/warpx_simf.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import time - -import numpy as np -from libensemble.executors.executor import Executor -from libensemble.message_numbers import TASK_FAILED, WORKER_DONE -from read_sim_output import read_sim_output -from write_sim_input import write_sim_input - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It defines a sim_f function that takes LibEnsemble history and -input parameters, run a WarpX simulation and returns 'f'. -""" - - -def run_warpx(H, persis_info, sim_specs, libE_info): - """ - This function runs a WarpX simulation and returns quantity 'f' as well as - other physical quantities measured in the run for convenience. Status check - is done periodically on the simulation, provided by LibEnsemble. - """ - - # Setting up variables needed for input and output - # keys = variable names - # x = variable values - # libE_output = what will be returned to libE - - calc_status = 0 # Returns to worker - - input_file = sim_specs['user']['input_filename'] - time_limit = sim_specs['user']['sim_kill_minutes'] * 60.0 - machine_specs = sim_specs['user']['machine_specs'] - - exctr = Executor.executor # Get Executor - - # Modify WarpX input file with input parameters calculated by gen_f - # and passed to this sim_f. - write_sim_input(input_file, H['x']) - - # Passed to command line in addition to the executable. - # Here, only input file - app_args = input_file - os.environ["OMP_NUM_THREADS"] = machine_specs['OMP_NUM_THREADS'] - - # Launch the executor to actually run the WarpX simulation - if machine_specs['name'] == 'summit': - task = exctr.submit(calc_type='sim', - extra_args=machine_specs['extra_args'], - app_args=app_args, - stdout='out.txt', - stderr='err.txt', - wait_on_run=True) - else: - task = exctr.submit(calc_type='sim', - num_procs=machine_specs['cores'], - app_args=app_args, - stdout='out.txt', - stderr='err.txt', - wait_on_run=True) - - # Periodically check the status of the simulation - poll_interval = 1 # secs - while(not task.finished): - time.sleep(poll_interval) - task.poll() - if task.runtime > time_limit: - task.kill() # Timeout - - # Set calc_status with optional prints. - if task.finished: - if task.state == 'FINISHED': - calc_status = WORKER_DONE - elif task.state == 'FAILED': - print("Warning: Task {} failed: Error code {}" - .format(task.name, task.errcode)) - calc_status = TASK_FAILED - elif task.state == 'USER_KILLED': - print("Warning: Task {} has been killed" - .format(task.name)) - else: - print("Warning: Task {} in unknown state {}. Error code {}" - .format(task.name, task.state, task.errcode)) - - # Safety - time.sleep(0.2) - - # Get output from a run and delete output files - warpx_out = read_sim_output(task.workdir) - - # Excluding results - NaN - from runs where beam was lost - if (warpx_out[0] != warpx_out[0]): - print(task.workdir, ' output led to NaN values (beam was lost or run did not finish)') - - # Pass the sim output values to LibEnsemble. - # When optimization is ON, 'f' is then passed to the generating function - # gen_f to generate new inputs for next runs. - # All other parameters are here just for convenience. - libE_output = np.zeros(1, dtype=sim_specs['out']) - libE_output['f'] = warpx_out[0] - libE_output['energy_std'] = warpx_out[1] - libE_output['energy_avg'] = warpx_out[2] - libE_output['charge'] = warpx_out[3] - libE_output['emittance'] = warpx_out[4] - libE_output['ramp_down_1'] = H['x'][0][0] - libE_output['ramp_down_2'] = H['x'][0][1] - libE_output['zlens_1'] = H['x'][0][2] - libE_output['adjust_factor'] = H['x'][0][3] - - return libE_output, persis_info, calc_status diff --git a/Tools/LibEnsemble/write_sim_input.py b/Tools/LibEnsemble/write_sim_input.py deleted file mode 100644 index e4a6398689e..00000000000 --- a/Tools/LibEnsemble/write_sim_input.py +++ /dev/null @@ -1,118 +0,0 @@ -import re - -import scipy.constants as scc - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It provides functions to modify some parameters in a WarpX -input file. -""" - - -def _set_value(str_text, str_line, val): - """ - This function search for str_line in str_text and overwrites the whole - matching line with str(val) - """ - str_text = re.sub('\n' + str_line + '.*', - '\n' + str_line + str(val), - str_text) - return str_text - - -def write_sim_input(input_file, x_values): - """ - This function modifies input_file to replace the values of some parameters - by values provided in x_values. - - Parameters - ---------- - input_file : WarpX input file. - x_values : Specific value of input x for this run. - """ - - # Parameters exposed to optimization - ramp_down_1 = x_values[0][0] - ramp_down_2 = x_values[0][1] - zlens_1 = x_values[0][2] - adjust_factor = x_values[0][3] - - # Fixed parameters - ramp_up_1 = 0.02 - plateau_1 = 0.297 - ramp_up_2 = ramp_up_1 - plateau_2 = plateau_1 - gap_12 = .0285 - - end_stage_1 = ramp_up_1 + plateau_1 + ramp_down_1 - beg_stage_2 = end_stage_1 + gap_12 - end_stage_2 = beg_stage_2 + ramp_up_2 + plateau_2 + ramp_down_2 - - # Updating mirrors position - mirror_z_1 = end_stage_1 + 1.0e-3 - mirror_z_2 = end_stage_2 + 1.0e-3 - - # Updating laser 2 configuration - z_antenna_2 = beg_stage_2 - 1.0e-9 #with value used for stage 1 - z_t_peak_2 = z_antenna_2 / scc.c + 2 * 7.33841e-14 # using pulse duration - - # End simulation when beam has just escaped the last stage - gamma_b = 30. - zmax_stop_run = end_stage_2 - 55.e-6 * gamma_b**2 * 2. - - with open(input_file) as file_handler: - output_text = file_handler.read() - - # Set end of stage 1 - output_text = _set_value( - output_text, 'electrons.zmax = ', str(end_stage_1)) - output_text = _set_value( - output_text, 'ions.zmax = ', str(end_stage_1)) - # Set length of final downramp of stage 1 - output_text = _set_value( - output_text, 'electrons.predefined_profile_params = ', - '0.0 .02 .297 ' + str(ramp_down_1) + ' 40.e-6 1.7e23') - output_text = _set_value( - output_text, 'ions.predefined_profile_params = ', - '0.0 .02 .297 ' + str(ramp_down_1) + ' 40.e-6 1.7e23') - # Set position of lens - output_text = _set_value( - output_text, 'my_constants.zlen = ', str(end_stage_1 + zlens_1)) - # Set beginning of stage 2 - output_text = _set_value( - output_text, 'electrons2.zmin = ', str(beg_stage_2)) - output_text = _set_value( - output_text, 'ions2.zmin = ', str(beg_stage_2)) - # Set end of stage 2 - output_text = _set_value( - output_text, 'electrons2.zmax = ', str(end_stage_2)) - output_text = _set_value( - output_text, 'ions2.zmax = ', str(end_stage_2)) - # Set length of final downramp of stage 2 - output_text = _set_value( - output_text, 'electrons2.predefined_profile_params = ', - str(beg_stage_2) + ' .02 .297 ' + str(ramp_down_2) + ' 40.e-6 1.7e23') - output_text = _set_value( - output_text, 'ions2.predefined_profile_params = ', - str(beg_stage_2) + ' .02 .297 ' + str(ramp_down_2) + ' 40.e-6 1.7e23') - # Set adjustment factor on lens strength - output_text = _set_value( - output_text, 'my_constants.adjust_factor = ', str(adjust_factor)) - # Set when to stop the run - output_text = _set_value( - output_text, 'warpx.zmax_plasma_to_compute_max_step = ', - str(zmax_stop_run)) - # Set mirrors position - output_text = _set_value( - output_text, 'warpx.mirror_z = ', str(mirror_z_1)+' '+str(mirror_z_2)) - # Set laser2 position - output_text = _set_value( - output_text, 'laser2.position = ', '0. 0. '+ str(z_antenna_2)) - # Set laser2 time until peak field is reached - output_text = _set_value( - output_text, 'laser2.profile_t_peak = ', str(z_t_peak_2)) - -# Write new input file - fout = open(input_file, 'w') - fout.write(output_text) - fout.close() From 5e95faf0659842c487c0400d4a882611806a027e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 12 Apr 2024 14:12:33 -0700 Subject: [PATCH 065/121] Always update guard cells in WarpXSumGuardCells (#4840) * Always update guard cells in WarpXSumGuardCells * Apply suggestions from code review * Update benchmarks * Update capacitive discharge test * Update benchmarks * Update Regression/WarpX-tests.ini * Update Examples/Tests/ohm_solver_EM_modes/analysis_rz.py * Update Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Update benchmark * Update box position * Update checksum --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../capacitive_discharge/analysis_1d.py | 66 +++++++++---------- .../capacitive_discharge/analysis_dsmc.py | 66 +++++++++---------- .../Tests/AcceleratorLattice/inputs_quad_3d | 4 +- .../Tests/ohm_solver_EM_modes/analysis_rz.py | 2 +- .../LaserAccelerationBoost.json | 42 ++++++------ .../LaserAcceleration_BTD.json | 50 +++++++------- .../LaserInjectionFromLASYFile_RZ.json | 14 ++-- .../LaserInjectionFromRZLASYFile.json | 16 ++--- .../benchmarks_json/Python_dsmc_1d.json | 38 +++++------ .../Python_ohms_law_solver_EM_modes_rz.json | 14 ++-- ...s_law_solver_magnetic_reconnection_2d.json | 20 +++--- .../Python_prev_positions.json | 24 +++---- .../hard_edged_quadrupoles.json | 24 +++---- .../benchmarks_json/magnetostatic_eb_3d.json | 32 ++++----- .../particle_thermal_boundary.json | 16 ++--- .../reduced_diags_single_precision.json | 54 +++++++-------- Source/Parallelization/WarpXSumGuardCells.H | 12 +--- Source/Parallelization/WarpXSumGuardCells.cpp | 18 +---- 18 files changed, 245 insertions(+), 267 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py index c6529dabd82..29b5272d8b1 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py @@ -5,39 +5,39 @@ import numpy as np ref_density = np.array([ - 1.27953969e+14, 2.23553999e+14, 2.55384510e+14, 2.55663110e+14, - 2.55805760e+14, 2.55812087e+14, 2.55813911e+14, 2.55754104e+14, - 2.55929601e+14, 2.56085472e+14, 2.55932867e+14, 2.55828121e+14, - 2.55901711e+14, 2.55985074e+14, 2.56182697e+14, 2.56446847e+14, - 2.56483696e+14, 2.56301187e+14, 2.56245301e+14, 2.56797584e+14, - 2.57257907e+14, 2.57023627e+14, 2.56500876e+14, 2.56106851e+14, - 2.56283546e+14, 2.56723967e+14, 2.56960855e+14, 2.56825486e+14, - 2.56674669e+14, 2.56567191e+14, 2.56310927e+14, 2.56361171e+14, - 2.56692197e+14, 2.56743606e+14, 2.56653108e+14, 2.56883854e+14, - 2.56763228e+14, 2.56343726e+14, 2.56385489e+14, 2.56570110e+14, - 2.56538112e+14, 2.56472179e+14, 2.56322922e+14, 2.56195384e+14, - 2.56474576e+14, 2.56764233e+14, 2.56533016e+14, 2.56257170e+14, - 2.56362463e+14, 2.56363962e+14, 2.56311292e+14, 2.56678788e+14, - 2.57061138e+14, 2.56785892e+14, 2.56406603e+14, 2.56334908e+14, - 2.56120051e+14, 2.56003269e+14, 2.56132187e+14, 2.56329572e+14, - 2.56535713e+14, 2.56708950e+14, 2.56661860e+14, 2.56448986e+14, - 2.56386823e+14, 2.56233660e+14, 2.56137632e+14, 2.56206263e+14, - 2.56364996e+14, 2.56483536e+14, 2.56308741e+14, 2.56447231e+14, - 2.56896301e+14, 2.56691405e+14, 2.56170780e+14, 2.56122216e+14, - 2.56427399e+14, 2.56897558e+14, 2.56928868e+14, 2.56659033e+14, - 2.56749993e+14, 2.56952497e+14, 2.56798907e+14, 2.56377081e+14, - 2.56453057e+14, 2.56796632e+14, 2.56944576e+14, 2.57248469e+14, - 2.57279426e+14, 2.56849516e+14, 2.56601834e+14, 2.56850545e+14, - 2.56953072e+14, 2.56442586e+14, 2.56329006e+14, 2.56790661e+14, - 2.57083582e+14, 2.57075550e+14, 2.56719615e+14, 2.56220486e+14, - 2.56222323e+14, 2.56547365e+14, 2.56499423e+14, 2.56434041e+14, - 2.56378587e+14, 2.56249892e+14, 2.56380492e+14, 2.56504513e+14, - 2.56337631e+14, 2.56204891e+14, 2.56325116e+14, 2.56297798e+14, - 2.56112782e+14, 2.56054218e+14, 2.56320120e+14, 2.56580938e+14, - 2.56446800e+14, 2.56267011e+14, 2.56372853e+14, 2.56617592e+14, - 2.56630745e+14, 2.56615242e+14, 2.56625259e+14, 2.56561320e+14, - 2.56640072e+14, 2.56693273e+14, 2.56613237e+14, 2.24169847e+14, - 1.27683197e+14 + 1.27989677e+14, 2.23601330e+14, 2.55400265e+14, 2.55664972e+14, + 2.55806841e+14, 2.55806052e+14, 2.55815865e+14, 2.55755151e+14, + 2.55920582e+14, 2.56078972e+14, 2.55933056e+14, 2.55840862e+14, + 2.56043990e+14, 2.56247464e+14, 2.56289153e+14, 2.56277109e+14, + 2.56242679e+14, 2.56328850e+14, 2.56373595e+14, 2.56395979e+14, + 2.56451842e+14, 2.56601614e+14, 2.56530735e+14, 2.56113125e+14, + 2.56207966e+14, 2.56474622e+14, 2.56378743e+14, 2.56230373e+14, + 2.56271563e+14, 2.56317460e+14, 2.56137709e+14, 2.56104767e+14, + 2.56244330e+14, 2.56188574e+14, 2.56070283e+14, 2.56212277e+14, + 2.56374214e+14, 2.56309276e+14, 2.56199679e+14, 2.56227549e+14, + 2.56434824e+14, 2.56578649e+14, 2.56418136e+14, 2.56781877e+14, + 2.57464056e+14, 2.57125505e+14, 2.56477914e+14, 2.56355343e+14, + 2.56599651e+14, 2.56955102e+14, 2.56988933e+14, 2.56954167e+14, + 2.56848084e+14, 2.56278229e+14, 2.55969967e+14, 2.56075186e+14, + 2.56011026e+14, 2.56054081e+14, 2.56203573e+14, 2.56131373e+14, + 2.56181410e+14, 2.56460942e+14, 2.56519893e+14, 2.56394874e+14, + 2.56273725e+14, 2.56263570e+14, 2.56463482e+14, 2.56541354e+14, + 2.56467160e+14, 2.56523576e+14, 2.56760359e+14, 2.56899283e+14, + 2.56889333e+14, 2.56984101e+14, 2.57009449e+14, 2.57017095e+14, + 2.57027915e+14, 2.57002409e+14, 2.56938087e+14, 2.56604366e+14, + 2.56392558e+14, 2.56481669e+14, 2.56337220e+14, 2.56107563e+14, + 2.56286144e+14, 2.56451638e+14, 2.56547674e+14, 2.56846908e+14, + 2.56725251e+14, 2.56321829e+14, 2.56387226e+14, 2.56750085e+14, + 2.56775478e+14, 2.56589827e+14, 2.56723109e+14, 2.56698688e+14, + 2.56674607e+14, 2.57128951e+14, 2.57144972e+14, 2.56602827e+14, + 2.56564415e+14, 2.56824494e+14, 2.56459673e+14, 2.56106618e+14, + 2.56442627e+14, 2.57106509e+14, 2.57468066e+14, 2.57155708e+14, + 2.56501993e+14, 2.56062276e+14, 2.56109738e+14, 2.56284272e+14, + 2.56320557e+14, 2.56253594e+14, 2.56155365e+14, 2.56044920e+14, + 2.55994560e+14, 2.56166612e+14, 2.56327633e+14, 2.56390898e+14, + 2.56474891e+14, 2.56669566e+14, 2.56646041e+14, 2.56264156e+14, + 2.56041610e+14, 2.56041551e+14, 2.56088641e+14, 2.23853646e+14, + 1.27580207e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index a7a76be46ad..94a08186183 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -18,39 +18,39 @@ my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) ref_density = np.array([ - 1.27957355e+14, 2.23554080e+14, 2.55373436e+14, 2.55659492e+14, - 2.55814670e+14, 2.55818418e+14, 2.55811882e+14, 2.55742272e+14, - 2.55912888e+14, 2.56086072e+14, 2.55944486e+14, 2.55830183e+14, - 2.55909337e+14, 2.56008609e+14, 2.56205930e+14, 2.56421940e+14, - 2.56369990e+14, 2.56151020e+14, 2.55925823e+14, 2.55924941e+14, - 2.56067211e+14, 2.56264104e+14, 2.56435035e+14, 2.56543804e+14, - 2.56715146e+14, 2.56639305e+14, 2.56509438e+14, 2.56478881e+14, - 2.56406748e+14, 2.56194832e+14, 2.56126186e+14, 2.56442221e+14, - 2.56603784e+14, 2.56592554e+14, 2.56475838e+14, 2.56304135e+14, - 2.56310993e+14, 2.56298883e+14, 2.56386742e+14, 2.56555670e+14, - 2.56588013e+14, 2.56851444e+14, 2.56928531e+14, 2.56637559e+14, - 2.56678652e+14, 2.56827322e+14, 2.56630197e+14, 2.56295404e+14, - 2.56285079e+14, 2.56558116e+14, 2.56676094e+14, 2.56577780e+14, - 2.56599749e+14, 2.56540500e+14, 2.56292984e+14, 2.56230350e+14, - 2.56363607e+14, 2.56553909e+14, 2.56501054e+14, 2.56249684e+14, - 2.56280268e+14, 2.56558208e+14, 2.56437837e+14, 2.56152650e+14, - 2.56143349e+14, 2.56067330e+14, 2.56020624e+14, 2.56039223e+14, - 2.56306096e+14, 2.56693084e+14, 2.56649778e+14, 2.56589778e+14, - 2.56594097e+14, 2.56368788e+14, 2.56290090e+14, 2.56420940e+14, - 2.56581419e+14, 2.56642649e+14, 2.56426887e+14, 2.56360122e+14, - 2.56573424e+14, 2.56679138e+14, 2.56488767e+14, 2.56217444e+14, - 2.56353118e+14, 2.56640765e+14, 2.56809490e+14, 2.56933226e+14, - 2.56633538e+14, 2.56203430e+14, 2.56202958e+14, 2.56564020e+14, - 2.56816347e+14, 2.56709830e+14, 2.56557382e+14, 2.56573904e+14, - 2.56745541e+14, 2.56784430e+14, 2.56580054e+14, 2.56210130e+14, - 2.56271415e+14, 2.56821160e+14, 2.56703292e+14, 2.56169296e+14, - 2.56166549e+14, 2.56467777e+14, 2.56573240e+14, 2.56437594e+14, - 2.56253730e+14, 2.56176123e+14, 2.56351125e+14, 2.56569916e+14, - 2.56761101e+14, 2.56891411e+14, 2.56628312e+14, 2.56180062e+14, - 2.56063564e+14, 2.56189728e+14, 2.56609454e+14, 2.57263643e+14, - 2.57097673e+14, 2.56666761e+14, 2.56622585e+14, 2.56432378e+14, - 2.56386718e+14, 2.56734491e+14, 2.57042448e+14, 2.24471147e+14, - 1.27720853e+14 + 1.27945114e+14, 2.23583642e+14, 2.55392166e+14, 2.55655614e+14, + 2.55811304e+14, 2.55813038e+14, 2.55817129e+14, 2.55752279e+14, + 2.55912208e+14, 2.56079483e+14, 2.55932916e+14, 2.55822911e+14, + 2.55910910e+14, 2.55995538e+14, 2.56185960e+14, 2.56518490e+14, + 2.56843281e+14, 2.56847244e+14, 2.56317496e+14, 2.55995033e+14, + 2.56036918e+14, 2.56185305e+14, 2.56473454e+14, 2.56612720e+14, + 2.56460349e+14, 2.56162495e+14, 2.56154687e+14, 2.56351568e+14, + 2.56518116e+14, 2.56589716e+14, 2.56491198e+14, 2.56462652e+14, + 2.56475832e+14, 2.56502009e+14, 2.56512082e+14, 2.56400671e+14, + 2.56317027e+14, 2.56348890e+14, 2.56604754e+14, 2.56846626e+14, + 2.56749617e+14, 2.56563946e+14, 2.56508159e+14, 2.56706913e+14, + 2.56921135e+14, 2.56772946e+14, 2.56509222e+14, 2.56556290e+14, + 2.56632138e+14, 2.56452482e+14, 2.56371644e+14, 2.56363151e+14, + 2.56471549e+14, 2.56541772e+14, 2.56342768e+14, 2.56345629e+14, + 2.56415143e+14, 2.56259609e+14, 2.56264787e+14, 2.56619762e+14, + 2.56992843e+14, 2.57102387e+14, 2.56719391e+14, 2.56255624e+14, + 2.56276638e+14, 2.56350832e+14, 2.56295294e+14, 2.56153556e+14, + 2.56170347e+14, 2.56246380e+14, 2.56088509e+14, 2.56121028e+14, + 2.56423889e+14, 2.56597963e+14, 2.56470310e+14, 2.56419968e+14, + 2.56625715e+14, 2.56711965e+14, 2.56651610e+14, 2.56750226e+14, + 2.56886322e+14, 2.56839170e+14, 2.56469219e+14, 2.56254127e+14, + 2.56422126e+14, 2.56381502e+14, 2.56283046e+14, 2.56358078e+14, + 2.56454503e+14, 2.56810744e+14, 2.57281306e+14, 2.57314168e+14, + 2.57216217e+14, 2.57166645e+14, 2.56969293e+14, 2.56763128e+14, + 2.56479604e+14, 2.56454515e+14, 2.56670498e+14, 2.56378555e+14, + 2.56177400e+14, 2.56510462e+14, 2.56512126e+14, 2.56391975e+14, + 2.56717395e+14, 2.57024288e+14, 2.56901669e+14, 2.56632950e+14, + 2.56470315e+14, 2.56530515e+14, 2.56839839e+14, 2.57084249e+14, + 2.56897794e+14, 2.56360159e+14, 2.56251337e+14, 2.56628550e+14, + 2.56743475e+14, 2.56616083e+14, 2.56830920e+14, 2.57164423e+14, + 2.56868182e+14, 2.56749114e+14, 2.57108990e+14, 2.56791445e+14, + 2.56203737e+14, 2.56114492e+14, 2.56630346e+14, 2.24843400e+14, + 1.28200732e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Tests/AcceleratorLattice/inputs_quad_3d b/Examples/Tests/AcceleratorLattice/inputs_quad_3d index 2bc98f64118..d62b9b45a0e 100644 --- a/Examples/Tests/AcceleratorLattice/inputs_quad_3d +++ b/Examples/Tests/AcceleratorLattice/inputs_quad_3d @@ -2,8 +2,8 @@ max_step = 50 amr.n_cell = 8 8 32 amr.max_level = 0 geometry.dims = 3 -geometry.prob_lo = -0.2 -0.2 0. -geometry.prob_hi = +0.2 +0.2 1.6 +geometry.prob_lo = -0.2 -0.2 -0.025 +geometry.prob_hi = +0.2 +0.2 1.575 warpx.const_dt = 1e-9 warpx.do_electrostatic = labframe diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index b96bc3b4629..de7c9966b20 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -153,7 +153,7 @@ def process(it): amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) print("Amplitude sample: ", amps) assert np.allclose( - amps, np.array([61.50945919, 19.74831134, 101.01820349, 10.8974811]) + amps, np.array([ 61.02377286, 19.80026021, 100.47687017, 10.83331295]) ) if sim.test: diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index 0efcdaeca3c..dd59536ad37 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 4818955.480792835, - "By": 1752.8025402207227, - "Bz": 14516.21278267981, - "Ex": 2366115496505.249, - "Ey": 1446112025634143.0, - "Ez": 21864189507353.19, - "jx": 1996366349839211.5, - "jy": 5.312583827165288e+16, - "jz": 2.049135262445976e+16, - "rho": 68443961.71835628 + "Bx": 4818955.480797943, + "By": 1752.8025638791275, + "Bz": 14516.212782554387, + "Ex": 2366115503598.9224, + "Ey": 1446112025635674.2, + "Ez": 21864189507357.867, + "jx": 1996366349775593.5, + "jy": 5.312583827155926e+16, + "jz": 2.0491352624508764e+16, + "rho": 68443961.71852128 }, "electrons": { - "particle_momentum_x": 2.2135945391227107e-23, - "particle_momentum_y": 2.8224559499572622e-22, - "particle_momentum_z": 5.260626010211241e-22, - "particle_position_x": 0.010800577787628053, + "particle_momentum_x": 2.2135945391319113e-23, + "particle_momentum_y": 2.8224559499558413e-22, + "particle_momentum_z": 5.260626010214114e-22, + "particle_position_x": 0.010800577787628052, "particle_position_y": 0.2111506062831815, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472277235318e-23, - "particle_momentum_y": 4.449097689427615e-22, - "particle_momentum_z": 5.768168724780326e-22, + "particle_momentum_x": 6.248472277246885e-23, + "particle_momentum_y": 4.449097689427654e-22, + "particle_momentum_z": 5.768168724998047e-22, "particle_position_x": 0.010800001678510512, "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 }, "beam": { - "particle_momentum_x": 3.535745635169933e-19, + "particle_momentum_x": 3.5357456351701565e-19, "particle_momentum_y": 4.363391839372122e-19, - "particle_momentum_z": 5.658606416951657e-17, - "particle_position_x": 0.008314723025211447, + "particle_momentum_z": 5.658606416951653e-17, + "particle_position_x": 0.008314723025211468, "particle_position_y": 1.1704335743854242, "particle_weight": 62415090744.60765 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index dd224516c5c..140bff16cc2 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497756265.44724107, - "By": 252197580.117894, - "Bz": 16988475.5444833, - "Ex": 7.299911104263803e+16, - "Ey": 1.460116488178734e+17, - "Ez": 2.26033100286114e+16, - "jx": 8.27644503757345e+17, - "jy": 2.1062078409959675e+18, - "jz": 2.8491727096438305e+19, - "rho": 91863764144.41415 - }, - "electrons": { - "particle_momentum_x": 5.76165226700654e-20, - "particle_momentum_y": 2.7567389504898156e-19, - "particle_momentum_z": 4.134562048099117e-19, - "particle_position_x": 0.0025269863605945427, - "particle_position_y": 0.0024538321295153346, - "particle_position_z": 0.17818421763751244, - "particle_weight": 1675789447169.5652 + "Bx": 499326907.0665159, + "By": 254256667.12630302, + "Bz": 16997157.572204895, + "Ex": 7.361789596479304e+16, + "Ey": 1.464987954436535e+17, + "Ez": 2.2588202735123868e+16, + "jx": 8.306469909158771e+17, + "jy": 2.1067947367705277e+18, + "jz": 2.86663445441587e+19, + "rho": 92447775221.08655 }, "beam": { - "particle_momentum_x": 1.1926313055043134e-17, - "particle_momentum_y": 2.7056205218547404e-17, - "particle_momentum_z": 5.1562131494813424e-14, - "particle_position_x": 0.000560821518739447, - "particle_position_y": 0.000800729816549036, - "particle_position_z": 2.9800048650570097, + "particle_momentum_x": 1.1926337276935442e-17, + "particle_momentum_y": 2.70596966890012e-17, + "particle_momentum_z": 5.1562142636293934e-14, + "particle_position_x": 0.0005608203524288058, + "particle_position_y": 0.000800752801704984, + "particle_position_z": 2.980004865036241, "particle_weight": 62415.090744607616 + }, + "electrons": { + "particle_momentum_x": 5.774426971193643e-20, + "particle_momentum_y": 2.742475325800203e-19, + "particle_momentum_z": 4.1461064946749695e-19, + "particle_position_x": 0.0025250307389502184, + "particle_position_y": 0.0024550212004677482, + "particle_position_z": 0.17819223528078157, + "particle_weight": 1675789447169.5652 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index 242cf00f0c3..379e0991819 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 100278645.72225758, - "Bz": 2509008.1146699786, - "Er": 3.263343388037509, - "Et": 3.0045297157982412e+16, - "Ez": 0.2422526411295923, - "jr": 49.67802097760541, - "jt": 4.3051397628943917e+17, + "Br": 100277153.12954171, + "Bz": 2508844.115818364, + "Er": 3.263693682174799, + "Et": 3.004484377567921e+16, + "Ez": 0.24219246674343975, + "jr": 49.68134418425658, + "jt": 4.305088291175302e+17, "jz": 0.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json index 8d58f5c9551..ef53dcabeec 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 193609031.89343664, - "Bz": 4689634.787639907, - "Er": 197280940689075.1, - "Et": 5.9716806429474664e+16, - "Ez": 1383812184076636.0, - "jr": 18582087468.75, - "jt": 1.193738117372394e+18, + "Br": 193606866.6957581, + "Bz": 4689375.7330573285, + "Er": 197284500110989.84, + "Et": 5.971613843782555e+16, + "Ez": 1383808350045244.5, + "jr": 18582922265.875, + "jt": 1.193688996955741e+18, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json index 0b38f78e6f9..67737738b2c 100644 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "rho_electrons": 0.00443743609125863, - "rho_he_ions": 0.005198801518328451 + "rho_electrons": 0.004437417901725754, + "rho_he_ions": 0.005200476260168854 + }, + "he_ions": { + "particle_momentum_x": 2.771800535001148e-19, + "particle_momentum_y": 2.759439998478633e-19, + "particle_momentum_z": 3.6231785185705813e-19, + "particle_position_x": 2201.7215060103235, + "particle_weight": 17191519531250.002 }, "neutrals": { - "particle_momentum_x": 1.404700281648976e-19, - "particle_momentum_y": 1.4028127127618884e-19, - "particle_momentum_z": 1.4090901433394346e-19, - "particle_position_x": 1120.7727446759352, + "particle_momentum_x": 1.4052816330675472e-19, + "particle_momentum_y": 1.4027759931535706e-19, + "particle_momentum_z": 1.4119910335756157e-19, + "particle_position_x": 1121.3642639505965, "particle_weight": 6.4588e+19 }, - "he_ions": { - "particle_momentum_x": 2.770386771117138e-19, - "particle_momentum_y": 2.7568040242914223e-19, - "particle_momentum_z": 3.619756966185903e-19, - "particle_position_x": 2200.683185473434, - "particle_weight": 17185500000000.002 - }, "electrons": { - "particle_momentum_x": 3.5129762363657864e-20, - "particle_momentum_y": 3.5431134517510143e-20, - "particle_momentum_z": 1.2592093336142964e-19, - "particle_position_x": 2142.0662480700303, - "particle_weight": 14593699218750.002 + "particle_momentum_x": 3.535219734698525e-20, + "particle_momentum_y": 3.5373517451957064e-20, + "particle_momentum_z": 1.2562184152660903e-19, + "particle_position_x": 2141.9783924023695, + "particle_weight": 14592914062500.002 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json index ca50554c394..ec1b6272092 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json @@ -1,12 +1,12 @@ { "lev=0": {}, "ions": { - "particle_momentum_x": 5.043858996017125e-17, - "particle_momentum_y": 5.0444743275098145e-17, - "particle_momentum_z": 5.05192925609973e-17, - "particle_position_x": 143164.42694236583, - "particle_position_y": 143166.51848290052, - "particle_theta": 2573261.754119082, - "particle_weight": 8.12868064536689e+18 + "particle_momentum_x": 5.0438993756415296e-17, + "particle_momentum_y": 5.0444406612873916e-17, + "particle_momentum_z": 5.0519292431385393e-17, + "particle_position_x": 143164.41713467025, + "particle_position_y": 143166.51845281923, + "particle_theta": 2573261.8729711357, + "particle_weight": 8.128680645366887e+18 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json index c7b467ad5f4..316bcb5b30d 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json @@ -2,17 +2,17 @@ "lev=0": { "Bx": 1524.8789585554075, "By": 639.8314135126764, - "Bz": 7.476000395458839, - "Ex": 56499935.20497879, - "Ey": 75068107.08471295, - "Ez": 309535670.18599623 + "Bz": 7.476064371022787, + "Ex": 56500024.54347144, + "Ey": 75064104.7364582, + "Ez": 309548482.989921 }, "ions": { - "particle_momentum_x": 7.14295522755638e-15, - "particle_momentum_y": 7.1380780610425e-15, - "particle_momentum_z": 7.141134469227045e-15, - "particle_position_x": 11170689.203149632, - "particle_position_y": 5585328.083196239, + "particle_momentum_x": 7.142955224072594e-15, + "particle_momentum_y": 7.13807805985974e-15, + "particle_momentum_z": 7.141134511815448e-15, + "particle_position_x": 11170689.202853566, + "particle_position_y": 5585328.08326772, "particle_weight": 9.036667901693183e+18 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_prev_positions.json b/Regression/Checksum/benchmarks_json/Python_prev_positions.json index 55c886775d5..8c7c6bf8b73 100644 --- a/Regression/Checksum/benchmarks_json/Python_prev_positions.json +++ b/Regression/Checksum/benchmarks_json/Python_prev_positions.json @@ -3,21 +3,21 @@ "Bx": 0.0, "By": 0.0, "Bz": 0.0, - "Ex": 3710588.849989976, + "Ex": 3710700.956616212, "Ey": 0.0, - "Ez": 646727.8074440088, - "jx": 15259.034603501308, - "jy": 650.139263398662, - "jz": 943.0244062246846 + "Ez": 646752.4642948855, + "jx": 15258.82611558372, + "jy": 650.13507310447, + "jz": 942.8394748261528 }, "electrons": { - "particle_momentum_x": 8.78764082600202e-23, + "particle_momentum_x": 8.787556176835159e-23, "particle_momentum_y": 3.78128494181519e-24, - "particle_momentum_z": 5.3969075504746586e-24, - "particle_position_x": 0.6227948811898449, - "particle_position_y": 0.5553787034074176, - "particle_prev_x": 0.6152547961491118, - "particle_prev_z": 0.5559363018627721, + "particle_momentum_z": 5.395758515337991e-24, + "particle_position_x": 0.6227736757162666, + "particle_position_y": 0.5553800210930071, + "particle_prev_x": 0.615237399194202, + "particle_prev_z": 0.5559373934585523, "particle_weight": 8569335937.5 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json index f8186748f2c..8a1ff9de966 100644 --- a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json +++ b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json @@ -3,20 +3,20 @@ "Bx": 0.0, "By": 0.0, "Bz": 0.0, - "Ex": 9.882421146615367e-06, - "Ey": 1.0440261046714249e-05, - "Ez": 1.003739697324731e-05, - "jx": 2.914866280957325e-10, - "jy": 8.46605718473121e-19, - "jz": 3.82349275686397e-08 + "Ex": 8.263130364346929e-06, + "Ey": 8.74492138450055e-06, + "Ez": 8.97527046111985e-06, + "jx": 2.9148662813789523e-10, + "jy": 8.513426949250112e-19, + "jz": 3.8234927568748794e-08 }, "electron": { - "particle_momentum_x": 2.0819392991319055e-25, - "particle_momentum_y": 6.046869894766189e-34, - "particle_momentum_z": 2.730924530729043e-23, - "particle_position_x": 0.03492328774658799, - "particle_position_y": 2.2742551618036812e-11, - "particle_position_z": 1.4915217664612082, + "particle_momentum_x": 2.0819392994331126e-25, + "particle_momentum_y": 6.080703685011498e-34, + "particle_momentum_z": 2.7309245307369126e-23, + "particle_position_x": 0.03492328774773231, + "particle_position_y": 2.3430381516062662e-11, + "particle_position_z": 1.491521766466965, "particle_weight": 1.0 } } diff --git a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json index 801611f748b..a1ec0b4c831 100644 --- a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json +++ b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json @@ -1,21 +1,21 @@ { - "beam": { - "particle_momentum_x": 1.356321653466651e-21, - "particle_momentum_y": 1.356321653466648e-21, - "particle_momentum_z": 7.150873477755126e-16, - "particle_position_x": 11163.999973966627, - "particle_position_y": 11163.999973966625, - "particle_position_z": 131662.5003103599, - "particle_weight": 20895107655113.465 - }, "lev=0": { - "Az": 11.35866332708226, - "Bx": 111.55929404955174, - "By": 111.55929404955177, - "Ex": 31268316751.31371, - "Ey": 31268316751.31371, - "jz": 1034841326.0482643, - "phi": 3144575598.841474, + "Az": 11.358663326449284, + "Bx": 111.55929407644248, + "By": 111.55929407644244, + "Ex": 31257180402.55472, + "Ey": 31257180402.55473, + "jz": 1034841325.9848926, + "phi": 3143521213.0157924, "rho": 3.449203918900721 + }, + "beam": { + "particle_momentum_x": 1.3604657334742729e-21, + "particle_momentum_y": 1.3604657334742772e-21, + "particle_momentum_z": 7.150873450281544e-16, + "particle_position_x": 11163.99997371537, + "particle_position_y": 11163.999973715368, + "particle_position_z": 131662.50031035842, + "particle_weight": 20895107655113.465 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json index 5c302d4c5b3..318aab0a1bc 100644 --- a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json +++ b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json @@ -1,12 +1,12 @@ { "lev=0": { - "Bx": 167.49421217759414, - "By": 269.6793406439766, - "Bz": 258.3282607081799, - "Ex": 864657516595.9229, - "Ey": 24072965021.088013, - "Ez": 702786324880.7338, - "divE": 5.137699003981513e+19, - "rho": 357057125.6699786 + "Bx": 153.42850019293041, + "By": 290.08173598472, + "Bz": 187.69788476927954, + "Ex": 703014828960.0149, + "Ey": 28192823505.416325, + "Ez": 693698719737.6497, + "divE": 4.868580396562553e+19, + "rho": 324055400.4292488 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json b/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json index 2944a6ae69a..d76d1ed7d4c 100644 --- a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json +++ b/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json @@ -1,27 +1,27 @@ { + "lev=0": { + "Bx": 0.08424138754164545, + "By": 0.08475193383267765, + "Bz": 0.08796438426563924, + "Ex": 106872541.64261818, + "Ey": 107805199.35060501, + "Ez": 107782783.64583969, + "jx": 728219.6885312796, + "jy": 736402.7559853494, + "jz": 734812.8777186275, + "rho": 0.02775080584090528, + "rho_electrons": 0.5250012277583664, + "rho_protons": 0.5250012273027096 + }, "electrons": { - "particle_momentum_x": 2.4417210113801336e-19, - "particle_momentum_y": 2.4564851493412593e-19, - "particle_momentum_z": 2.4341594564719795e-19, - "particle_position_x": 16379.247754118871, - "particle_position_y": 16382.836191849463, - "particle_position_z": 16382.833818260406, + "particle_momentum_x": 2.4417210857855684e-19, + "particle_momentum_y": 2.4564845792611e-19, + "particle_momentum_z": 2.434159547126453e-19, + "particle_position_x": 16379.247745715009, + "particle_position_y": 16382.83609924704, + "particle_position_z": 16382.833801764602, "particle_weight": 800000003014656.0 }, - "lev=0": { - "Bx": 0.084241235376183, - "By": 0.084751961809296, - "Bz": 0.08796431812220362, - "Ex": 106872513.1172905, - "Ey": 107805221.85876846, - "Ez": 107782819.9927826, - "jx": 728219.9166720062, - "jy": 736402.6373479366, - "jz": 734813.3440347463, - "rho": 0.0277508111536644, - "rho_electrons": 0.52500122784204, - "rho_protons": 0.5250012273481843 - }, "photons": { "particle_momentum_x": 1.428291590249666e-18, "particle_momentum_y": 1.4222174024686332e-18, @@ -32,12 +32,12 @@ "particle_weight": 800000003014656.0 }, "protons": { - "particle_momentum_x": 1.4104799077996007e-19, - "particle_momentum_y": 1.4120357666424943e-19, - "particle_momentum_z": 1.3903168923764969e-19, - "particle_position_x": 16383.951009266078, - "particle_position_y": 16383.991230053827, - "particle_position_z": 16384.033095981926, + "particle_momentum_x": 1.4104797829545176e-19, + "particle_momentum_y": 1.4120353509003028e-19, + "particle_momentum_z": 1.3903169103008315e-19, + "particle_position_x": 16383.951009267941, + "particle_position_y": 16383.991229955107, + "particle_position_z": 16384.03309576772, "particle_weight": 800000003014656.0 } -} +} \ No newline at end of file diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 260ebb3871a..9c7c2e60160 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -16,11 +16,7 @@ * This is typically called for the sources of the Maxwell equations (J/rho) * after deposition from the macroparticles. * - * - When WarpX is used with a finite-difference scheme: this only - * updates the *valid* cells of `mf` - * - When WarpX is used with a spectral scheme (PSATD): this - * updates both the *valid* cells and *guard* cells. (This is because a - * spectral solver requires the value of the sources over a large stencil.) + * This updates both the *valid* cells and *guard* cells. */ void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, @@ -33,11 +29,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * This is typically called for the sources of the Maxwell equations (J/rho) * after deposition from the macroparticles + filtering. * - * - When WarpX is used with a finite-difference scheme: this only - * updates the *valid* cells of `dst` - * - When WarpX is used with a spectral scheme (PSATD): this - * updates both the *valid* cells and *guard* cells. (This is because a - * spectral solver requires the value of the sources over a large stencil.) + * This updates both the *valid* cells and *guard* cells. * * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp index 20ac73cab50..e752d49eedf 100644 --- a/Source/Parallelization/WarpXSumGuardCells.cpp +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -19,14 +19,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, const int icomp, const int ncomp) { - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = mf.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } + amrex::IntVect const n_updated_guards = mf.nGrowVect(); ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); } @@ -37,14 +30,7 @@ WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::IntVect& src_ngrow, const int icomp, const int ncomp) { - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = dst.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } + amrex::IntVect const n_updated_guards = dst.nGrowVect(); dst.setVal(0., icomp, ncomp, n_updated_guards); dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); From 12349a8d6cd08e5497e7ccf27f6004e4e58464ef Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Mon, 15 Apr 2024 09:27:09 -0700 Subject: [PATCH 066/121] Set OpenMP threads manually on Perlmutter nodes (#4857) --- Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch | 1 + Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch | 1 + 2 files changed, 2 insertions(+) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch index a1687b2a3b4..d13c7e3b4e5 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch @@ -33,6 +33,7 @@ INPUTS=inputs_small export SRUN_CPUS_PER_TASK=16 # 8 cores per chiplet, 2x SMP export OMP_PLACES=threads export OMP_PROC_BIND=spread +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} srun --cpu-bind=cores \ ${EXE} ${INPUTS} \ diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch index 873fd30355f..f2ea5fa3e7f 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch @@ -34,6 +34,7 @@ export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank # note: 16 avoids hyperthreading (32 virtual cores, 16 physical) export SRUN_CPUS_PER_TASK=16 +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" From 58e6b8def4769fb18f9c7467b89f8418b587eebb Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 15 Apr 2024 22:18:20 +0200 Subject: [PATCH 067/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4858) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f47dd01e057..06eb90b4395 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - + cd ../amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index d1c98b315a0..a54553689ad 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 +branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 12fed1e8bb9..f35a7676ab8 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 +branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 3f2987ec9cd..9d21519ce71 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "2a3955a5f5aac1aef6e6e72687f182331e049c39" +set(WarpX_amrex_branch "7813bb7d6bc6cea658edac1f2bde046c7d174168" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index d4c488f9113..be0e857f6f5 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "9b0a4d82bc14756530efe4e0c9ce3463ca5df742" +set(WarpX_pyamrex_branch "a9e0188ac11865f395a01f8b3195dace67b1f8e8" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 1d95dd5f2d2..774a220dc06 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - +cd amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 3add0096b1d7fa0030fd35b1fca25f140fde33df Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:54:44 -0700 Subject: [PATCH 068/121] Optimizing GPU acceleration of binary collision algorithms. (#4577) * opt gpu accel of particle collisions * minor updates in .gitignore * fix for consistent and correct particle collision error * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * minor bug fixing * [minor]: remove stale code and clean up comments * undo erroneously pushed changes in GNUmakefile * undo erroneously pushed inputs_3d * revert errors from obsolete code plus minor improvements * fix for a possible segfault * replace std::min with amrex::min for future compatibility * clang-tidy updates * Update random seed * Update benchmark files * Update .gitignore Co-authored-by: Remi Lehe * Update .gitignore Co-authored-by: Remi Lehe * Use bisect function * Simplify the code * Update benchmark and fix style * use `BinaryCollision` class for DSMC collisions * comment out explicit delete of `ScatteringProcess` constructor * fix first GPU compilation issue * fix RZ issue * avoid copying `binary_collision_functor` * fix merge error * do not keep `ScatteringProcess` instances in `DSMCFunc` * only keep array of `ScatteringProcess::Executor`s * fix clang-tidy errors and reset benchmark * fix default collision type issue * copy cross-section data to `DSMCFunc` * update checksum after fixing issue with cross-section arrays * move logic to set up cross-section interpolators to `DSMCFunc.cpp` file * use internal `struct`s in collision filter functors * clean-up and fixing clang-tidy issues * second clang-tidy fix attempt * update `DSMCFunc` with new collision pair parallelization strategy * fix DSMC CI benchmark file values * Fix clang-tidy error * update reference density array for DSMC CI test * Only call redistribute if collision type has product species * fix clang-tidy error * another clang-tidy error fix * Appropriately set `local` parameter in `Redistribute` call. * set `Redistribute` arguments appropriately * check if weight is numerically zero * Apply suggestions from code review Co-authored-by: Remi Lehe * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add docstring for `CollisionPairFilter` function arguments * Use `insert` rather than `push_back` & `rotate` Co-authored-by: Remi Lehe * apply suggestions from a few of the review comments * reduce differences between `SplitAndScatterFunc.H` and `ParticleCreationFunc.H` * use `deleteInvalidParticles` from #4849 * avoid unnecessary duplication when species1 == species2 * Apply suggestions from code review * Apply suggestions from code review * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: roelof-groenewald Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../capacitive_discharge/analysis_dsmc.py | 66 +++---- .../inputs_deuterium_tritium_3d | 2 + .../Deuterium_Deuterium_Fusion_3D.json | 40 ++--- ...rium_Deuterium_Fusion_3D_intraspecies.json | 44 ++--- .../Deuterium_Tritium_Fusion_3D.json | 114 ++++++------ .../Deuterium_Tritium_Fusion_RZ.json | 110 ++++++------ .../Proton_Boron_Fusion_2D.json | 144 +++++++-------- .../Proton_Boron_Fusion_3D.json | 164 +++++++++--------- .../benchmarks_json/Python_dsmc_1d.json | 34 ++-- .../benchmarks_json/collisionISO.json | 18 +- .../BinaryCollision/BinaryCollision.H | 136 +++++++++++++-- .../Coulomb/ElasticCollisionPerez.H | 24 ++- .../Coulomb/PairWiseCoulombCollisionFunc.H | 5 +- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 25 ++- .../DSMC/SplitAndScatterFunc.H | 6 +- .../NuclearFusion/NuclearFusionFunc.H | 25 ++- .../BinaryCollision/ParticleCreationFunc.H | 12 +- 17 files changed, 551 insertions(+), 418 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index 94a08186183..4c46b9e3f10 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -18,39 +18,39 @@ my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) ref_density = np.array([ - 1.27945114e+14, 2.23583642e+14, 2.55392166e+14, 2.55655614e+14, - 2.55811304e+14, 2.55813038e+14, 2.55817129e+14, 2.55752279e+14, - 2.55912208e+14, 2.56079483e+14, 2.55932916e+14, 2.55822911e+14, - 2.55910910e+14, 2.55995538e+14, 2.56185960e+14, 2.56518490e+14, - 2.56843281e+14, 2.56847244e+14, 2.56317496e+14, 2.55995033e+14, - 2.56036918e+14, 2.56185305e+14, 2.56473454e+14, 2.56612720e+14, - 2.56460349e+14, 2.56162495e+14, 2.56154687e+14, 2.56351568e+14, - 2.56518116e+14, 2.56589716e+14, 2.56491198e+14, 2.56462652e+14, - 2.56475832e+14, 2.56502009e+14, 2.56512082e+14, 2.56400671e+14, - 2.56317027e+14, 2.56348890e+14, 2.56604754e+14, 2.56846626e+14, - 2.56749617e+14, 2.56563946e+14, 2.56508159e+14, 2.56706913e+14, - 2.56921135e+14, 2.56772946e+14, 2.56509222e+14, 2.56556290e+14, - 2.56632138e+14, 2.56452482e+14, 2.56371644e+14, 2.56363151e+14, - 2.56471549e+14, 2.56541772e+14, 2.56342768e+14, 2.56345629e+14, - 2.56415143e+14, 2.56259609e+14, 2.56264787e+14, 2.56619762e+14, - 2.56992843e+14, 2.57102387e+14, 2.56719391e+14, 2.56255624e+14, - 2.56276638e+14, 2.56350832e+14, 2.56295294e+14, 2.56153556e+14, - 2.56170347e+14, 2.56246380e+14, 2.56088509e+14, 2.56121028e+14, - 2.56423889e+14, 2.56597963e+14, 2.56470310e+14, 2.56419968e+14, - 2.56625715e+14, 2.56711965e+14, 2.56651610e+14, 2.56750226e+14, - 2.56886322e+14, 2.56839170e+14, 2.56469219e+14, 2.56254127e+14, - 2.56422126e+14, 2.56381502e+14, 2.56283046e+14, 2.56358078e+14, - 2.56454503e+14, 2.56810744e+14, 2.57281306e+14, 2.57314168e+14, - 2.57216217e+14, 2.57166645e+14, 2.56969293e+14, 2.56763128e+14, - 2.56479604e+14, 2.56454515e+14, 2.56670498e+14, 2.56378555e+14, - 2.56177400e+14, 2.56510462e+14, 2.56512126e+14, 2.56391975e+14, - 2.56717395e+14, 2.57024288e+14, 2.56901669e+14, 2.56632950e+14, - 2.56470315e+14, 2.56530515e+14, 2.56839839e+14, 2.57084249e+14, - 2.56897794e+14, 2.56360159e+14, 2.56251337e+14, 2.56628550e+14, - 2.56743475e+14, 2.56616083e+14, 2.56830920e+14, 2.57164423e+14, - 2.56868182e+14, 2.56749114e+14, 2.57108990e+14, 2.56791445e+14, - 2.56203737e+14, 2.56114492e+14, 2.56630346e+14, 2.24843400e+14, - 1.28200732e+14 + 1.27943881e+14, 2.23583097e+14, 2.55396716e+14, 2.55673406e+14, + 2.55827566e+14, 2.55803446e+14, 2.55798707e+14, 2.55748961e+14, + 2.55906413e+14, 2.56063991e+14, 2.55937018e+14, 2.55841390e+14, + 2.55917724e+14, 2.55988641e+14, 2.56052050e+14, 2.56285151e+14, + 2.56647960e+14, 2.56756264e+14, 2.56430158e+14, 2.56117493e+14, + 2.56065302e+14, 2.56265220e+14, 2.56328575e+14, 2.56031495e+14, + 2.56123757e+14, 2.56431173e+14, 2.56385320e+14, 2.56391170e+14, + 2.56561177e+14, 2.56513926e+14, 2.56332201e+14, 2.56252442e+14, + 2.56238982e+14, 2.56216498e+14, 2.56461281e+14, 2.56863199e+14, + 2.56908100e+14, 2.56926112e+14, 2.57001641e+14, 2.56735963e+14, + 2.56315358e+14, 2.56137028e+14, 2.56101418e+14, 2.56276827e+14, + 2.56425668e+14, 2.56181798e+14, 2.56044925e+14, 2.56330387e+14, + 2.56623150e+14, 2.56445316e+14, 2.56292750e+14, 2.56440918e+14, + 2.56433406e+14, 2.56186982e+14, 2.56236390e+14, 2.56469557e+14, + 2.56349704e+14, 2.56487457e+14, 2.56771823e+14, 2.56614683e+14, + 2.56552210e+14, 2.56850291e+14, 2.56783396e+14, 2.56483187e+14, + 2.56510868e+14, 2.56490408e+14, 2.56656042e+14, 2.56820924e+14, + 2.56640314e+14, 2.56465063e+14, 2.56510264e+14, 2.56917331e+14, + 2.57228490e+14, 2.56960593e+14, 2.56587911e+14, 2.56672682e+14, + 2.56774414e+14, 2.56548335e+14, 2.56225540e+14, 2.56079693e+14, + 2.56062796e+14, 2.56054612e+14, 2.56028683e+14, 2.56068820e+14, + 2.56380975e+14, 2.56654914e+14, 2.56776792e+14, 2.56983661e+14, + 2.56989477e+14, 2.56646250e+14, 2.56589639e+14, 2.56946205e+14, + 2.57091201e+14, 2.56913590e+14, 2.56513535e+14, 2.56122597e+14, + 2.56176340e+14, 2.56808001e+14, 2.57239393e+14, 2.56845066e+14, + 2.56662482e+14, 2.56862583e+14, 2.56518922e+14, 2.56155531e+14, + 2.56362794e+14, 2.57203564e+14, 2.57737938e+14, 2.57252026e+14, + 2.56859277e+14, 2.56658995e+14, 2.56357364e+14, 2.56393454e+14, + 2.56714308e+14, 2.57042200e+14, 2.57551087e+14, 2.57502490e+14, + 2.56641118e+14, 2.56401115e+14, 2.56644629e+14, 2.56673096e+14, + 2.56534659e+14, 2.56357745e+14, 2.56455309e+14, 2.56586850e+14, + 2.56442415e+14, 2.56335971e+14, 2.56411429e+14, 2.24109018e+14, + 1.27678869e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d index 0943eee1d22..8099d92928b 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d @@ -11,6 +11,8 @@ geometry.dims = 3 geometry.prob_lo = 0. 0. 0. geometry.prob_hi = 8. 8. 16. +warpx.random_seed = 4 + ################################# ###### Boundary Condition ####### ################################# diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json index 0d426c07ed9..76d03f521b8 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json @@ -1,4 +1,7 @@ { + "lev=0": { + "rho": 0.0 + }, "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -8,15 +11,6 @@ "particle_position_z": 81919021.52308556, "particle_weight": 1024.000000000021 }, - "helium3_1": { - "particle_momentum_x": 8.492383942294212e-16, - "particle_momentum_y": 8.507905928154269e-16, - "particle_momentum_z": 8.500197251270117e-16, - "particle_position_x": 151779.74563236872, - "particle_position_y": 152858.5501479658, - "particle_position_z": 322785.7613257161, - "particle_weight": 2.7065032055210726e-28 - }, "hydrogen2_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -26,16 +20,22 @@ "particle_position_z": 81920546.19181262, "particle_weight": 1024.000000000021 }, - "lev=0": { - "rho": 0.0 - }, "neutron_1": { - "particle_momentum_x": 8.492383942294212e-16, - "particle_momentum_y": 8.507905928154269e-16, - "particle_momentum_z": 8.500197251270117e-16, - "particle_position_x": 151779.74563236872, - "particle_position_y": 152858.5501479658, - "particle_position_z": 322785.7613257161, - "particle_weight": 2.7065032055210726e-28 + "particle_momentum_x": 8.574995119045322e-16, + "particle_momentum_y": 8.563358069072292e-16, + "particle_momentum_z": 8.571142005592407e-16, + "particle_position_x": 152851.7089469153, + "particle_position_y": 151779.17287974653, + "particle_position_z": 325950.1658699999, + "particle_weight": 2.7360848012854956e-28 + }, + "helium3_1": { + "particle_momentum_x": 8.574995119045322e-16, + "particle_momentum_y": 8.563358069072292e-16, + "particle_momentum_z": 8.571142005592407e-16, + "particle_position_x": 152851.7089469153, + "particle_position_y": 151779.17287974653, + "particle_position_z": 325950.1658699999, + "particle_weight": 2.7360848012854956e-28 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json index 1c6af40e173..59818f62826 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json @@ -1,4 +1,18 @@ { + "lev=0": { + "rho": 0.0, + "rho_deuterium": 8203144355.71195, + "rho_helium3": 10.368009592276463 + }, + "neutron": { + "particle_momentum_x": 2.2543499835759282e-15, + "particle_momentum_y": 2.2526527390783875e-15, + "particle_momentum_z": 2.2619641737859965e-15, + "particle_position_x": 61.961041864660686, + "particle_position_y": 61.78141653674165, + "particle_position_z": 61.741022731492514, + "particle_weight": 505562702.7678892 + }, "deuterium": { "particle_momentum_x": 1.3370046499332103e-14, "particle_momentum_y": 1.3364310231320824e-14, @@ -6,29 +20,15 @@ "particle_position_x": 2560.1613417364665, "particle_position_y": 2560.082464065988, "particle_position_z": 2560.0018477161034, - "particle_weight": 7.999999989895393e+17 + "particle_weight": 7.999999989888742e+17 }, "helium3": { - "particle_momentum_x": 2.2800935865848323e-15, - "particle_momentum_y": 2.2864983116288316e-15, - "particle_momentum_z": 2.290487090201156e-15, - "particle_position_x": 62.07326392385665, - "particle_position_y": 62.074468969610855, - "particle_position_z": 62.07087662970013, - "particle_weight": 505230602.2536906 - }, - "lev=0": { - "rho": 0.0, - "rho_deuterium": 8203144355.71876, - "rho_helium3": 10.36119892112141 - }, - "neutron": { - "particle_momentum_x": 2.2625878931428226e-15, - "particle_momentum_y": 2.2652096393691827e-15, - "particle_momentum_z": 2.27105060559148e-15, - "particle_position_x": 62.07326392385665, - "particle_position_y": 62.074468969610855, - "particle_position_z": 62.07087662970013, - "particle_weight": 505230602.2536906 + "particle_momentum_x": 2.2749239620327265e-15, + "particle_momentum_y": 2.268697031603961e-15, + "particle_momentum_z": 2.278045756364995e-15, + "particle_position_x": 61.961041864660686, + "particle_position_y": 61.78141653674165, + "particle_position_z": 61.741022731492514, + "particle_weight": 505562702.7678892 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json index 23231eab789..1ac85583f9d 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json @@ -2,76 +2,76 @@ "lev=0": { "rho": 0.0 }, - "neutron_1": { - "particle_momentum_x": 1.7519716491839538e-15, - "particle_momentum_y": 1.7523289312260283e-15, - "particle_momentum_z": 1.7480231586369996e-15, - "particle_position_x": 154379.32401483235, - "particle_position_y": 152618.63815943015, - "particle_position_z": 325970.4138010667, - "particle_weight": 4.421535775967805e-28 + "neutron_2": { + "particle_momentum_x": 1.5428541105411113e-15, + "particle_momentum_y": 1.5521530475782452e-15, + "particle_momentum_z": 1.5581061911002605e-15, + "particle_position_x": 139420.85239440785, + "particle_position_y": 137544.4080412421, + "particle_position_z": 292987.6112145397, + "particle_weight": 6.353652226479104e+18 }, - "tritium_1": { + "tritium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 + "particle_momentum_z": 0.0, + "particle_position_x": 409665.266470154, + "particle_position_y": 409535.84596852644, + "particle_position_z": 819126.8984535293, + "particle_weight": 1.0239999999364636e+29 }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.356220762633467e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503, - "particle_weight": 1.0240001137713503e+30 + "helium4_2": { + "particle_momentum_x": 1.5428541105411113e-15, + "particle_momentum_y": 1.5521530475782452e-15, + "particle_momentum_z": 1.7807114780425105e-15, + "particle_position_x": 139420.85239440785, + "particle_position_y": 137544.4080412421, + "particle_position_z": 292987.6112145397, + "particle_weight": 6.353652226479104e+18 }, - "helium4_1": { - "particle_momentum_x": 1.7519716491839538e-15, - "particle_momentum_y": 1.7523289312260283e-15, - "particle_momentum_z": 1.7480231586369996e-15, - "particle_position_x": 154379.32401483235, - "particle_position_y": 152618.63815943015, - "particle_position_z": 325970.4138010667, - "particle_weight": 4.421535775967805e-28 + "neutron_1": { + "particle_momentum_x": 1.7286463849660049e-15, + "particle_momentum_y": 1.7253692489143775e-15, + "particle_momentum_z": 1.7397872072929338e-15, + "particle_position_x": 151971.91531558792, + "particle_position_y": 150436.3752328273, + "particle_position_z": 320653.7895132215, + "particle_weight": 4.360021412453755e-28 }, "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, + "particle_position_x": 40958427.509923086, + "particle_position_y": 40959476.344507605, + "particle_position_z": 81921930.27522033, "particle_weight": 1024.000000000021 }, - "neutron_2": { - "particle_momentum_x": 1.538345593941914e-15, - "particle_momentum_y": 1.536969107959402e-15, - "particle_momentum_z": 1.5535605933245691e-15, - "particle_position_x": 137088.2335716813, - "particle_position_y": 136370.55273167047, - "particle_position_z": 290148.26756873244, - "particle_weight": 6.427760042646557e+18 - }, - "helium4_2": { - "particle_momentum_x": 1.538345593941914e-15, - "particle_momentum_y": 1.536969107959402e-15, - "particle_momentum_z": 1.769309377628039e-15, - "particle_position_x": 137088.2335716813, - "particle_position_y": 136370.55273167047, - "particle_position_z": 290148.26756873244, - "particle_weight": 6.427760042646557e+18 + "tritium_1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.8875978729147693e-13, + "particle_position_x": 40959200.1108159, + "particle_position_y": 40960650.40789148, + "particle_position_z": 81920772.79861207, + "particle_weight": 1024.000000000021 }, - "tritium_2": { + "deuterium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999357226e+29 + "particle_momentum_z": 3.356220762633467e-14, + "particle_position_x": 4096177.559084946, + "particle_position_y": 4096353.028787281, + "particle_position_z": 8192362.405430989, + "particle_weight": 1.024000113771424e+30 + }, + "helium4_1": { + "particle_momentum_x": 1.7286463849660049e-15, + "particle_momentum_y": 1.7253692489143775e-15, + "particle_momentum_z": 1.7397872072929338e-15, + "particle_position_x": 151971.91531558792, + "particle_position_y": 150436.3752328273, + "particle_position_z": 320653.7895132215, + "particle_weight": 4.360021412453755e-28 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json index 3fb0db8d8a5..bccb0a12b0f 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json @@ -1,64 +1,28 @@ { - "deuterium_1": { - "particle_momentum_x": 1.8388106511899905e-13, - "particle_momentum_y": 1.837868790009435e-13, - "particle_momentum_z": 0.0, - "particle_position_x": 40959919.499819286, - "particle_position_y": 81919224.48541151, - "particle_theta": 32166860.23003994, - "particle_weight": 3216.984554806547 - }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.336364094249911e-14, - "particle_position_x": 4095908.9083809257, - "particle_position_y": 8192069.080030457, - "particle_theta": 3216444.348910214, - "particle_weight": 3.1898417901971444e+29 - }, - "helium4_1": { - "particle_momentum_x": 1.858124399143442e-15, - "particle_momentum_y": 1.876715110797694e-15, - "particle_momentum_z": 1.7098432207359157e-15, - "particle_position_x": 152920.23233108618, - "particle_position_y": 323733.9138644398, - "particle_theta": 120064.13771707338, - "particle_weight": 1.603083276067953e-27 - }, - "helium4_2": { - "particle_momentum_x": 1.5195006688950936e-15, - "particle_momentum_y": 1.52430083815551e-15, - "particle_momentum_z": 1.7654865863613367e-15, - "particle_position_x": 136867.63803188328, - "particle_position_y": 286903.30393175944, - "particle_theta": 107912.20520382549, - "particle_weight": 2.0862696876352987e+19 - }, "lev=0": { "rho": 0.0 }, - "neutron_1": { - "particle_momentum_x": 1.7160671487712845e-15, - "particle_momentum_y": 1.7154753069055672e-15, - "particle_momentum_z": 1.7098432207359157e-15, - "particle_position_x": 152920.23233108618, - "particle_position_y": 323733.9138644398, - "particle_theta": 120064.13771707338, - "particle_weight": 1.603083276067953e-27 + "helium4_2": { + "particle_momentum_x": 1.500608194743691e-15, + "particle_momentum_y": 1.5109305701135078e-15, + "particle_momentum_z": 1.7349325969148392e-15, + "particle_position_x": 134553.8162997425, + "particle_position_y": 285988.79476631305, + "particle_theta": 106450.92963352974, + "particle_weight": 2.0493998275383095e+19 }, "neutron_2": { - "particle_momentum_x": 1.5195006688950936e-15, - "particle_momentum_y": 1.52430083815551e-15, - "particle_momentum_z": 1.5463311225724366e-15, - "particle_position_x": 136867.63803188328, - "particle_position_y": 286903.30393175944, - "particle_theta": 107912.20520382549, - "particle_weight": 2.0862696876352987e+19 + "particle_momentum_x": 1.500608194743691e-15, + "particle_momentum_y": 1.5109305701135078e-15, + "particle_momentum_z": 1.5208565520173107e-15, + "particle_position_x": 134553.8162997425, + "particle_position_y": 285988.79476631305, + "particle_theta": 106450.92963352974, + "particle_weight": 2.0493998275383095e+19 }, "tritium_1": { - "particle_momentum_x": 1.8384658063720362e-13, - "particle_momentum_y": 1.8381593257898129e-13, + "particle_momentum_x": 1.8384658063720355e-13, + "particle_momentum_y": 1.8381593257898126e-13, "particle_momentum_z": 0.0, "particle_position_x": 40961278.052658774, "particle_position_y": 81919046.8061561, @@ -72,6 +36,42 @@ "particle_position_x": 409793.9651940968, "particle_position_y": 819237.3558155322, "particle_theta": 321974.4557387621, - "particle_weight": 3.218514276139388e+29 + "particle_weight": 3.218514276143075e+29 + }, + "helium4_1": { + "particle_momentum_x": 1.8900919654269547e-15, + "particle_momentum_y": 1.88323376518843e-15, + "particle_momentum_z": 1.7117554385103313e-15, + "particle_position_x": 154816.5295407665, + "particle_position_y": 326401.3426220656, + "particle_theta": 121193.01462929095, + "particle_weight": 1.615304706407927e-27 + }, + "deuterium_1": { + "particle_momentum_x": 1.8388106511899905e-13, + "particle_momentum_y": 1.837868790009435e-13, + "particle_momentum_z": 0.0, + "particle_position_x": 40959919.499819286, + "particle_position_y": 81919224.48541151, + "particle_theta": 32166860.23003994, + "particle_weight": 3216.984554806547 + }, + "neutron_1": { + "particle_momentum_x": 1.7399029833189488e-15, + "particle_momentum_y": 1.7342017841614303e-15, + "particle_momentum_z": 1.7117554385103313e-15, + "particle_position_x": 154816.5295407665, + "particle_position_y": 326401.3426220656, + "particle_theta": 121193.01462929095, + "particle_weight": 1.615304706407927e-27 + }, + "deuterium_2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 3.336364094249911e-14, + "particle_position_x": 4095908.9083809257, + "particle_position_y": 8192069.080030457, + "particle_theta": 3216444.348910214, + "particle_weight": 3.18984179020083e+29 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index 7f359f76e94..c6bd572ab68 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -2,53 +2,29 @@ "lev=0": { "rho": 0.0 }, - "alpha2": { - "particle_momentum_x": 4.0815251136381854e-15, - "particle_momentum_y": 4.122335955939925e-15, - "particle_momentum_z": 4.201726051973563e-15, - "particle_position_x": 410664.0477768091, - "particle_position_y": 868193.1483606595, - "particle_weight": 2.947401325355292e+18 - }, - "alpha3": { - "particle_momentum_x": 4.925995964082186e-16, - "particle_momentum_y": 4.780894181553476e-16, - "particle_momentum_z": 4.688680302272935e-16, - "particle_position_x": 52240.870920562535, - "particle_position_y": 103873.30862578771, - "particle_weight": 1.4784234459335885e+27 - }, - "proton1": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512789535533e+28 }, "alpha4": { - "particle_momentum_x": 2.3422553081132076e-14, - "particle_momentum_y": 2.3416860254545166e-14, - "particle_momentum_z": 2.3532380279126124e-14, - "particle_position_x": 2457367.4582781536, + "particle_momentum_x": 2.3381598715816888e-14, + "particle_momentum_y": 2.3435337108040707e-14, + "particle_momentum_z": 2.3554073734365495e-14, + "particle_position_x": 2457367.4582781526, "particle_position_y": 4915112.044373058, "particle_weight": 384.0000000000002 }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 - }, "boron3": { "particle_momentum_x": 9.277692671587846e-15, "particle_momentum_y": 9.268409636965691e-15, "particle_momentum_z": 9.279446607709548e-15, "particle_position_x": 4096178.1664224654, "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399507192184686e+30 + "particle_weight": 6.399508749004609e+30 }, "boron2": { "particle_momentum_x": 0.0, @@ -56,23 +32,47 @@ "particle_momentum_z": 0.0, "particle_position_x": 409798.015821768, "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999999017534e+28 + "particle_weight": 1.279999999903553e+28 }, - "boron1": { + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 + }, + "alpha2": { + "particle_momentum_x": 4.057984510682467e-15, + "particle_momentum_y": 4.104188139725855e-15, + "particle_momentum_z": 4.17858000090827e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 2.893405947366684e+18 }, "proton3": { - "particle_momentum_x": 1.684673285246867e-15, - "particle_momentum_y": 1.6827557106531144e-15, - "particle_momentum_z": 1.6802642612723895e-15, - "particle_position_x": 2457259.537951346, - "particle_position_y": 4914248.771185745, - "particle_weight": 1.2795071921846893e+30 + "particle_momentum_x": 1.6847528610914633e-15, + "particle_momentum_y": 1.6827441855152735e-15, + "particle_momentum_z": 1.6802807054711977e-15, + "particle_position_x": 2457284.001605452, + "particle_position_y": 4914327.937654163, + "particle_weight": 1.2795087490046132e+30 + }, + "alpha3": { + "particle_momentum_x": 4.969030674549252e-16, + "particle_momentum_y": 4.889174794354074e-16, + "particle_momentum_z": 4.826711561221412e-16, + "particle_position_x": 52379.50835085382, + "particle_position_y": 104954.74986555413, + "particle_weight": 1.4737529861620076e+27 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 }, "proton5": { "particle_momentum_x": 0.0, @@ -82,36 +82,36 @@ "particle_position_y": 819101.3225783394, "particle_weight": 1.2800000000000004e+37 }, - "alpha1": { - "particle_momentum_x": 4.714228774936194e-15, - "particle_momentum_y": 4.676079523757908e-15, - "particle_momentum_z": 4.676143086393951e-15, - "particle_position_x": 463834.9497057527, - "particle_position_y": 977592.6452214213, - "particle_weight": 2.9280275175431764e-28 - }, - "alpha5": { - "particle_momentum_x": 2.3300446729308862e-14, - "particle_momentum_y": 2.3449941485701153e-14, - "particle_momentum_z": 2.3618372938672865e-14, - "particle_position_x": 2457556.8571638414, - "particle_position_y": 4914659.635379325, - "particle_weight": 3.839999999999998e-19 - }, - "proton2": { + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789517532e+28 + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, + "particle_weight": 128.00000000000261 }, - "boron5": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, + "particle_weight": 128.00000000000261 + }, + "alpha1": { + "particle_momentum_x": 4.691909092431811e-15, + "particle_momentum_y": 4.6836958163275755e-15, + "particle_momentum_z": 4.657203546376977e-15, + "particle_position_x": 463465.27131109464, + "particle_position_y": 978207.6061359186, + "particle_weight": 2.964832324596836e-28 + }, + "alpha5": { + "particle_momentum_x": 2.3343469003376247e-14, + "particle_momentum_y": 2.3450435277836452e-14, + "particle_momentum_z": 2.357999284036709e-14, + "particle_position_x": 2457556.8571638423, + "particle_position_y": 4914659.635379323, + "particle_weight": 3.839999999999998e-19 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index c3517c32b5b..9a078d754c3 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -2,68 +2,32 @@ "lev=0": { "rho": 0.0 }, - "proton3": { - "particle_momentum_x": 1.684809640261926e-15, - "particle_momentum_y": 1.6828399494797829e-15, - "particle_momentum_z": 1.6804535487104095e-15, - "particle_position_x": 2457338.899376694, - "particle_position_y": 2457069.647393952, - "particle_position_z": 4914642.288898885, - "particle_weight": 1.0236580897818594e+31 - }, - "alpha5": { - "particle_momentum_x": 2.3336421176154494e-14, - "particle_momentum_y": 2.345583787205298e-14, - "particle_momentum_z": 2.3574859187827772e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 2457059.6353793233, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119658089781855e+31 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 + "alpha2": { + "particle_momentum_x": 4.1179548991012315e-15, + "particle_momentum_y": 4.110026665992801e-15, + "particle_momentum_z": 4.169802553223462e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 1.901545867395127e+19 }, "alpha4": { - "particle_momentum_x": 2.3374367002622734e-14, - "particle_momentum_y": 2.3439200330576348e-14, - "particle_momentum_z": 2.3559200621107925e-14, - "particle_position_x": 2457367.4582781536, + "particle_momentum_x": 2.3389474647479255e-14, + "particle_momentum_y": 2.3425362750821152e-14, + "particle_momentum_z": 2.3515097401900785e-14, + "particle_position_x": 2457367.458278154, "particle_position_y": 2457512.044373057, - "particle_position_z": 4914475.776513074, + "particle_position_z": 4914475.776513075, "particle_weight": 3072.000000000002 }, - "alpha3": { - "particle_momentum_x": 4.857656981301579e-16, - "particle_momentum_y": 4.811177053359803e-16, - "particle_momentum_z": 4.766749863500389e-16, - "particle_position_x": 51282.9136300544, - "particle_position_y": 51045.39533309579, - "particle_position_z": 101578.82008857065, - "particle_weight": 1.0257306544231767e+28 - }, - "alpha2": { - "particle_momentum_x": 4.071664828713061e-15, - "particle_momentum_y": 4.119627260272026e-15, - "particle_momentum_z": 4.181812341443065e-15, - "particle_position_x": 405848.1200755021, - "particle_position_y": 408011.53191288817, - "particle_position_z": 866330.5923068221, - "particle_weight": 1.9180757241165136e+19 + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 }, "proton2": { "particle_momentum_x": 0.0, @@ -72,7 +36,16 @@ "particle_position_x": 4095630.698135355, "particle_position_y": 4096073.5517983637, "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240716198e+29 + "particle_weight": 1.0227810240721708e+29 + }, + "proton3": { + "particle_momentum_x": 1.6847690263373399e-15, + "particle_momentum_y": 1.682851264092124e-15, + "particle_momentum_z": 1.6803794796027466e-15, + "particle_position_x": 2457363.4111393224, + "particle_position_y": 2457049.324917686, + "particle_position_z": 4914549.051577593, + "particle_weight": 1.0236635064099502e+31 }, "proton4": { "particle_momentum_x": 0.0, @@ -83,14 +56,23 @@ "particle_position_z": 818958.0399127571, "particle_weight": 1.0240000000000003e+38 }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 + "alpha5": { + "particle_momentum_x": 2.3387550695963677e-14, + "particle_momentum_y": 2.33445389960837e-14, + "particle_momentum_z": 2.3635948557453833e-14, + "particle_position_x": 2457556.8571638414, + "particle_position_y": 2457059.6353793237, + "particle_position_z": 4915847.043341329, + "particle_weight": 3.0719999999999984e-18 + }, + "alpha3": { + "particle_momentum_x": 4.72203056468584e-16, + "particle_momentum_y": 4.622060921861793e-16, + "particle_momentum_z": 4.5443204949049515e-16, + "particle_position_x": 50603.134609459696, + "particle_position_y": 48681.319634796026, + "particle_position_z": 100449.24933492322, + "particle_weight": 1.0094807701511393e+28 }, "proton5": { "particle_momentum_x": 0.0, @@ -101,23 +83,14 @@ "particle_position_z": 819309.1804186807, "particle_weight": 1.0240000000000003e+38 }, - "boron5": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 - }, - "alpha1": { - "particle_momentum_x": 4.659632773478301e-15, - "particle_momentum_y": 4.617309192789389e-15, - "particle_momentum_z": 4.6173901462667306e-15, - "particle_position_x": 453219.3436004627, - "particle_position_y": 457393.42114332493, - "particle_position_z": 970481.0580153911, - "particle_weight": 1.8966442168729786e-27 + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 }, "boron2": { "particle_momentum_x": 0.0, @@ -126,6 +99,33 @@ "particle_position_x": 409798.0158217681, "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, - "particle_weight": 1.023999999936064e+29 + "particle_weight": 1.0239999999366152e+29 + }, + "alpha1": { + "particle_momentum_x": 4.665933695243743e-15, + "particle_momentum_y": 4.603805875733438e-15, + "particle_momentum_z": 4.706765986105302e-15, + "particle_position_x": 461871.79172011977, + "particle_position_y": 461162.2166206925, + "particle_position_z": 969262.7809050508, + "particle_weight": 1.9171578359058453e-27 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.119663506409945e+31 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json index 67737738b2c..53c3708056f 100644 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "rho_electrons": 0.004437417901725754, - "rho_he_ions": 0.005200476260168854 + "rho_electrons": 0.004434311371276535, + "rho_he_ions": 0.005200489146365628 }, "he_ions": { - "particle_momentum_x": 2.771800535001148e-19, - "particle_momentum_y": 2.759439998478633e-19, - "particle_momentum_z": 3.6231785185705813e-19, - "particle_position_x": 2201.7215060103235, - "particle_weight": 17191519531250.002 + "particle_momentum_x": 2.7688639005236794e-19, + "particle_momentum_y": 2.760648864116014e-19, + "particle_momentum_z": 3.6226628630061563e-19, + "particle_position_x": 2201.614485497906, + "particle_weight": 17190996093750.002 }, "neutrals": { - "particle_momentum_x": 1.4052816330675472e-19, - "particle_momentum_y": 1.4027759931535706e-19, - "particle_momentum_z": 1.4119910335756157e-19, - "particle_position_x": 1121.3642639505965, + "particle_momentum_x": 1.4040775167811838e-19, + "particle_momentum_y": 1.4009514702703257e-19, + "particle_momentum_z": 1.4093144247152345e-19, + "particle_position_x": 1119.524782452131, "particle_weight": 6.4588e+19 }, "electrons": { - "particle_momentum_x": 3.535219734698525e-20, - "particle_momentum_y": 3.5373517451957064e-20, - "particle_momentum_z": 1.2562184152660903e-19, - "particle_position_x": 2141.9783924023695, - "particle_weight": 14592914062500.002 + "particle_momentum_x": 3.5193566020597954e-20, + "particle_momentum_y": 3.5368803263788353e-20, + "particle_momentum_z": 1.2572273121326582e-19, + "particle_position_x": 2139.3709122461873, + "particle_weight": 14579566406250.002 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 742cb775230..96a2f5cbdac 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -1,13 +1,4 @@ { - "electron": { - "particle_momentum_x": 3.5796522874263304e-19, - "particle_momentum_y": 3.580087349086981e-19, - "particle_momentum_z": 3.5805012694309815e-19, - "particle_position_x": 1.024116253197213, - "particle_position_y": 1.0238635904620974, - "particle_position_z": 1.0240133505121767, - "particle_weight": 714240000000.0 - }, "lev=0": { "Bx": 0.0, "By": 0.0, @@ -18,5 +9,14 @@ "jx": 0.0, "jy": 0.0, "jz": 0.0 + }, + "electron": { + "particle_momentum_x": 3.5856774369040155e-19, + "particle_momentum_y": 3.5832349891760786e-19, + "particle_momentum_z": 3.5757013442445486e-19, + "particle_position_x": 1.0241562532152744, + "particle_position_y": 1.0238555904782611, + "particle_position_z": 1.0240053504978093, + "particle_weight": 714240000000.0 } } \ No newline at end of file diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 807dc827b6b..bd6ec84dd69 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -174,7 +174,7 @@ public: #endif if (m_have_product_species){ species1.defineAllParticleTiles(); - species2.defineAllParticleTiles(); + if (!m_isSameSpecies) { species2.defineAllParticleTiles(); } } // Enable tiling @@ -214,7 +214,7 @@ public: // the colliding species (`species1` and `species2`) may be removed // (i.e., marked as invalid) in the process of creating new product particles. species1.deleteInvalidParticles(); - species2.deleteInvalidParticles(); + if (!m_isSameSpecies) { species2.deleteInvalidParticles(); } } } } @@ -325,6 +325,25 @@ public: p_n_pairs_in_each_cell, pair_offsets.data()); index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + amrex::Gpu::DeviceVector n_ind_pairs_in_each_cell(n_cells+1); + index_type* AMREX_RESTRICT p_n_ind_pairs_in_each_cell = n_ind_pairs_in_each_cell.dataPtr(); + + amrex::ParallelFor( n_cells+1, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell = (i_cell < n_cells)? cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]: 0; + // number of independent collisions in each cell + p_n_ind_pairs_in_each_cell[i_cell] = n_part_in_cell/2; + } + ); + + // start indices of independent collisions. + amrex::Gpu::DeviceVector coll_offsets(n_cells+1); + // number of total independent collision pairs + const auto n_independent_pairs = (int) amrex::Scan::ExclusiveSum(n_cells+1, + p_n_ind_pairs_in_each_cell, coll_offsets.data(), amrex::Scan::RetSum{true}); + index_type* AMREX_RESTRICT p_coll_offsets = coll_offsets.dataPtr(); + // mask: equal to 1 if particle creation occurs for a given pair, 0 otherwise amrex::Gpu::DeviceVector mask(n_total_pairs); index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); @@ -343,7 +362,6 @@ public: End of calculations only required when creating product particles */ - // Loop over cells amrex::ParallelForRNG( n_cells, [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept @@ -354,16 +372,44 @@ public: index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; - // Same but for the pairs - index_type const cell_start_pair = have_product_species? - p_pair_offsets[i_cell] : 0; - // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( indices_1, cell_start_1, cell_half_1, engine ); + } + ); + + // Loop over independent particle pairs + // To speed up binary collisions on GPU, we try to expose as much parallelism + // as possible (while avoiding race conditions): Instead of looping with one GPU + // thread per cell, we loop with one GPU thread per "independent pairs" (i.e. pairs + // that do not touch the same macroparticles, so that there is no race condition), + // where the number of independent pairs is determined by the lower number of + // macroparticles of either species, within each cell. + amrex::ParallelForRNG( n_independent_pairs, + [=] AMREX_GPU_DEVICE (int i_coll, amrex::RandomEngine const& engine) noexcept + { + // to avoid type mismatch errors + auto ui_coll = (index_type)i_coll; + + // Use a bisection algorithm to find the index of the cell in which this pair is located + const int i_cell = amrex::bisect( p_coll_offsets, 0, n_cells, ui_coll ); + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; + + // collision number of the cell + const index_type coll_idx = ui_coll - p_coll_offsets[i_cell]; + + // Same but for the pairs + index_type const cell_start_pair = have_product_species? + p_pair_offsets[i_cell] : 0; + #if defined WARPX_DIM_RZ const int ri = (i_cell - i_cell%nz) / nz; auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; @@ -376,9 +422,9 @@ public: cell_half_1, cell_stop_1, indices_1, indices_1, soa_1, soa_1, get_position_1, get_position_1, - q1, q1, m1, m1, dt, dV, + q1, q1, m1, m1, dt, dV, coll_idx, cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, engine ); + p_pair_reaction_weight, engine); } ); @@ -444,7 +490,6 @@ public: auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); #endif - /* The following calculations are only required when creating product particles */ @@ -477,6 +522,32 @@ public: p_n_pairs_in_each_cell, pair_offsets.data()); index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + amrex::Gpu::DeviceVector n_ind_pairs_in_each_cell(n_cells+1); + index_type* AMREX_RESTRICT p_n_ind_pairs_in_each_cell = n_ind_pairs_in_each_cell.dataPtr(); + + amrex::ParallelFor( n_cells+1, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + if (i_cell < n_cells) + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + p_n_ind_pairs_in_each_cell[i_cell] = amrex::min(n_part_in_cell_1, n_part_in_cell_2); + } + else + { + p_n_ind_pairs_in_each_cell[i_cell] = 0; + } + } + ); + + // start indices of independent collisions. + amrex::Gpu::DeviceVector coll_offsets(n_cells+1); + // number of total independent collision pairs + const auto n_independent_pairs = (int) amrex::Scan::ExclusiveSum(n_cells+1, + p_n_ind_pairs_in_each_cell, coll_offsets.data(), amrex::Scan::RetSum{true}); + index_type* AMREX_RESTRICT p_coll_offsets = coll_offsets.dataPtr(); + // mask: equal to 1 if particle creation occurs for a given pair, 0 otherwise amrex::Gpu::DeviceVector mask(n_total_pairs); index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); @@ -507,9 +578,6 @@ public: // Same for species 2 index_type const cell_start_2 = cell_offsets_2[i_cell]; index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; - // Same but for the pairs - index_type const cell_start_pair = have_product_species? - p_pair_offsets[i_cell] : 0; // ux from species1 can be accessed like this: // ux_1[ indices_1[i] ], where i is between @@ -522,6 +590,44 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); + } + ); + + // Loop over independent particle pairs + // To speed up binary collisions on GPU, we try to expose as much parallelism + // as possible (while avoiding race conditions): Instead of looping with one GPU + // thread per cell, we loop with one GPU thread per "independent pairs" (i.e. pairs + // that do not touch the same macroparticles, so that there is no race condition), + // where the number of independent pairs is determined by the lower number of + // macroparticles of either species, within each cell. + amrex::ParallelForRNG( n_independent_pairs, + [=] AMREX_GPU_DEVICE (int i_coll, amrex::RandomEngine const& engine) noexcept + { + // to avoid type mismatch errors + auto ui_coll = (index_type)i_coll; + + // Use a bisection algorithm to find the index of the cell in which this pair is located + const int i_cell = amrex::bisect( p_coll_offsets, 0, n_cells, ui_coll ); + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + + // collision number of the cell + const index_type coll_idx = ui_coll - p_coll_offsets[i_cell]; + + // Same but for the pairs + index_type const cell_start_pair = have_product_species? + p_pair_offsets[i_cell]: 0; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + #if defined WARPX_DIM_RZ const int ri = (i_cell - i_cell%nz) / nz; auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; @@ -533,9 +639,9 @@ public: cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, indices_1, indices_2, soa_1, soa_2, get_position_1, get_position_2, - q1, q2, m1, m2, dt, dV, + q1, q2, m1, m2, dt, dV, coll_idx, cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, engine ); + p_pair_reaction_weight, engine); } ); diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index d7403eeacf9..e407ae1603b 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -37,6 +37,7 @@ * @param[in] dV is the volume of the corresponding cell. * @param[in] engine the random number generator state & factory * @param[in] isSameSpecies whether this is an intra-species collision process + * @param[in] coll_idx is the collision index offset. */ template @@ -52,10 +53,12 @@ void ElasticCollisionPerez ( T_PR const T1, T_PR const T2, T_R const dt, T_PR const L, T_R const dV, amrex::RandomEngine const& engine, - bool const isSameSpecies) + bool const isSameSpecies, T_index coll_idx) { const T_index NI1 = I1e - I1s; const T_index NI2 = I2e - I2s; + const T_index max_N = amrex::max(NI1,NI2); + const T_index min_N = amrex::max(NI1,NI2); T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; @@ -95,7 +98,7 @@ void ElasticCollisionPerez ( n1 = n1 / dV; n2 = n2 / dV; { T_index i1 = I1s; T_index i2 = I2s; - for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) + for (T_index k = 0; k < max_N; ++k) { n12 += amrex::min( w1[ I1[i1] ], w2[ I2[i2] ] ); ++i1; if ( i1 == I1e ) { i1 = I1s; } @@ -126,10 +129,13 @@ void ElasticCollisionPerez ( // call UpdateMomentumPerezElastic() { - T_index i1 = I1s; T_index i2 = I2s; - for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) - { + T_index i1 = I1s + coll_idx; + T_index i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (T_index k = coll_idx; k < max_N; k += min_N) + { #if (defined WARPX_DIM_RZ) /* In RZ geometry, macroparticles can collide with other macroparticles * in the same *cylindrical* cell. For this reason, collisions between macroparticles @@ -159,8 +165,12 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); #endif - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index b31fa3922b9..79fe1d42b7d 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -76,6 +76,7 @@ public: * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. * @param[in] engine the random engine. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE @@ -88,7 +89,7 @@ public: GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const /*cell_start_pair*/, index_type* /*p_mask*/, index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, amrex::ParticleReal* /*p_pair_reaction_weight*/, @@ -100,7 +101,7 @@ public: I1s, I1e, I2s, I2e, I1, I2, soa_1, soa_2, q1, q2, m1, m2, -1.0_prt, -1.0_prt, - dt, m_CoulombLog, dV, engine, m_isSameSpecies); + dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); } amrex::ParticleReal m_CoulombLog; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 1039ee96969..3c3849580d1 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -75,6 +75,7 @@ public: * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. * @param[in] cell_start_pair is the start index of the pairs in that cell. * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given * pair. It is only needed here to store information that will be used later on when actually @@ -97,7 +98,7 @@ public: GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, @@ -117,10 +118,9 @@ public: const index_type NI1 = I1e - I1s; const index_type NI2 = I2e - I2s; const index_type max_N = amrex::max(NI1,NI2); + const index_type min_N = amrex::min(NI1,NI2); - index_type i1 = I1s; - index_type i2 = I2s; - index_type pair_index = cell_start_pair; + index_type pair_index = cell_start_pair + coll_idx; // Because the number of particles of each species is not always equal (NI1 != NI2 // in general), some macroparticles will be paired with multiple macroparticles of the @@ -135,8 +135,11 @@ public: amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; #endif - - for (index_type k = 0; k < max_N; ++k) + index_type i1 = I1s + coll_idx; + index_type i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (index_type k = coll_idx; k < max_N; k += min_N) { // c1k : how many times the current particle of species 1 is paired with a particle // of species 2. Same for c2k. @@ -175,9 +178,13 @@ public: p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } - ++pair_index; + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } + pair_index += min_N; } } diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index 074226d8227..a44ab90c4cc 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -162,7 +162,7 @@ public: // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) { #if defined(AMREX_USE_OMP) #pragma omp atomic write @@ -179,11 +179,11 @@ public: // setting its id to invalid if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); } if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); } // Set the child particle properties appropriately diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b3826114b09..a7dbc67b862 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -110,6 +110,7 @@ public: * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. * @param[in] cell_start_pair is the start index of the pairs in that cell. * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given * pair. It is only needed here to store information that will be used later on when actually @@ -132,7 +133,7 @@ public: GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, @@ -152,10 +153,9 @@ public: const index_type NI1 = I1e - I1s; const index_type NI2 = I2e - I2s; const index_type max_N = amrex::max(NI1,NI2); + const index_type min_N = amrex::min(NI1,NI2); - index_type i1 = I1s; - index_type i2 = I2s; - index_type pair_index = cell_start_pair; + index_type pair_index = cell_start_pair + coll_idx; // Because the number of particles of each species is not always equal (NI1 != NI2 // in general), some macroparticles will be paired with multiple macroparticles of the @@ -174,8 +174,11 @@ public: amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; #endif - - for (index_type k = 0; k < max_N; ++k) + index_type i1 = I1s + coll_idx; + index_type i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (index_type k = coll_idx; k < max_N; k += min_N) { // c1k : how many times the current particle of species 1 is paired with a particle // of species 2. Same for c2k. @@ -215,9 +218,13 @@ public: p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } - ++pair_index; + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } + pair_index += min_N; } } diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 4a77006dd92..580ddf57290 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -204,7 +204,7 @@ public: // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) { #if defined(AMREX_USE_OMP) #pragma omp atomic write @@ -218,15 +218,15 @@ public: }; // If the colliding particle weight decreases to zero, remove particle by - // setting its id to -1 - if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) + // setting its id to invalid + if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); } - if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) + if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); } // Initialize the product particles' momentum, using a function depending on the From 6be423ee57f9da98d03531a1500e919543902a5a Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:22:05 -0700 Subject: [PATCH 069/121] Add thermal boundary specification to picmi (#4850) * add thermal boundary specification to picmi.py * include thermal boundary handling for Cartesian grids --- Python/pywarpx/picmi.py | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 5902b958bc9..4bf9d4b4163 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -602,6 +602,11 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -629,6 +634,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -653,6 +661,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles pywarpx.boundary.reflect_all_velocities = self.reflect_all_velocities + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -706,6 +718,11 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -730,6 +747,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -746,6 +766,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -807,6 +831,11 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -833,6 +862,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -851,6 +883,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -928,6 +964,11 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -956,6 +997,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -976,6 +1020,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: From 83d801726c457d3c5ab72bbb5d11b8da5131f3f3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 16 Apr 2024 12:46:08 -0700 Subject: [PATCH 070/121] Add M. Haseeb in the list of developers (#4862) * Add M. Haseeb in the list of developers * Update .zenodo.json Co-authored-by: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> --------- Co-authored-by: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 5431e40c8bb..a2c9b443f99 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -98,6 +98,11 @@ "name": "Gu, Junmin", "orcid": "0000-0002-1521-8534" }, + { + "affiliation": "Lawrence Berkeley National Laboratory", + "name": "Haseeb, Muhammad", + "orcid": "0000-0002-0697-6894" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Jambunathan, Revathi", From 3b18ed71d00efcd29e65b6b8f6722ff106a9e5f7 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 18 Apr 2024 07:44:14 +0200 Subject: [PATCH 071/121] Update pyAMReX (#4870) Pull in the latest `development`. --- cmake/dependencies/pyAMReX.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index be0e857f6f5..038b98c191b 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "a9e0188ac11865f395a01f8b3195dace67b1f8e8" +set(WarpX_pyamrex_branch "28897d26f00707ee33d2423b8c24a020cd0767e0" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From a082660a73731658398bb8e027148d82342e46b2 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:24:07 -0700 Subject: [PATCH 072/121] Add function to set value of `do_not_push` (#4868) * Add function to set value of `do_no_push` * clang-tidy fix --- Source/Particles/WarpXParticleContainer.H | 4 +++- Source/Python/Particles/WarpXParticleContainer.cpp | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 61a87a12830..d4c325fbfb8 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -426,6 +426,8 @@ public: virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} #endif + void setDoNotPush (bool flag) { do_not_push = flag; } + protected: int species_id; @@ -442,7 +444,7 @@ protected: //! instead of gathering fields from the finest patch level, gather from the coarsest bool m_gather_from_main_grid = false; - int do_not_push = 0; + bool do_not_push = false; int do_not_gather = 0; // Whether to allow particles outside of the simulation domain to be diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 3feae5d8c90..aa2cd7a2091 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -137,5 +137,9 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("lev"), py::arg("local") ) + .def("set_do_not_push", + [](WarpXParticleContainer& pc, bool flag) { pc.setDoNotPush(flag); }, + py::arg("flag") + ) ; } From fc94b2341a3a567af471585f58478a58ebd1dbc0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 22 Apr 2024 16:09:04 -0700 Subject: [PATCH 073/121] Downgrade Reg Tests (#4882) Downgrade the regression testing scripts to work around https://github.com/AMReX-Codes/regression_testing/issues/136 --- run_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run_test.sh b/run_test.sh index 774a220dc06..2848cce415d 100755 --- a/run_test.sh +++ b/run_test.sh @@ -80,6 +80,8 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git +# FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 +cd regression_testing && git checkout db67b6bd80cff208dba59ec2938ffd02e8ad9cef && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks From 60e9b8e0ca7ca1bdc2c7e845f46a5eeb8ba55afe Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 01:17:47 +0200 Subject: [PATCH 074/121] remove unnecessary include of WarpX.H (#4877) --- .../Particles/Collision/BinaryCollision/ParticleCreationFunc.H | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 580ddf57290..83312017e44 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -15,7 +15,6 @@ #include "Particles/ParticleCreation/SmartCopy.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" -#include "WarpX.H" #include #include From 1d3c837bdab502fa17d8b379773d3bc0c42fcfe0 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 01:18:13 +0200 Subject: [PATCH 075/121] Ensure that only necessary headers are included in FieldAccessorFunctors.H (#4878) --- .../FiniteDifferenceAlgorithms/FieldAccessorFunctors.H | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H index d4fdf207e52..05b1db1fe94 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H @@ -8,11 +8,9 @@ #ifndef WARPX_FIELD_ACCESSOR_FUNCTORS_H #define WARPX_FIELD_ACCESSOR_FUNCTORS_H -#include "WarpX.H" -#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" - #include - +#include +#include /** * \brief Functor that returns the division of the source m_field Array4 value From b900941c295456141bc00a2aef543cb3b04488fb Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 01:19:11 +0200 Subject: [PATCH 076/121] remove unnecessary include from FlushFormat.H (#4879) --- Source/Diagnostics/FlushFormats/FlushFormat.H | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index 65741e4ff20..be1322bd61b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -5,7 +5,6 @@ #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/MultiParticleContainer.H" -#include "WarpX.H" class FlushFormat { From be379d21bcd963e789e608bbf035513ffc331a6a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 18:05:37 +0200 Subject: [PATCH 077/121] Further downgrade regression_testing (#4884) --- run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_test.sh b/run_test.sh index 2848cce415d..23602682ff5 100755 --- a/run_test.sh +++ b/run_test.sh @@ -81,7 +81,7 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git # FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 -cd regression_testing && git checkout db67b6bd80cff208dba59ec2938ffd02e8ad9cef && cd - +cd regression_testing && git checkout 93ddfb11456f47d6555c39388ba1a4ead61fbf4e && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks From cceae549a76a92482314d6685396077f1cff8a6d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 23 Apr 2024 10:06:01 -0700 Subject: [PATCH 078/121] DepositCharge: Component 1 Allocated? (#4859) * DepositCharge: Component 1 Allocated? * Move assert Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Source/Particles/PhysicalParticleContainer.cpp | 2 ++ Source/Particles/WarpXParticleContainer.cpp | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4b2d73c7d6a..e979afa9f77 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2228,6 +2228,8 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit charge after particle push, in component 1 of MultiFab rho. // (Skipped for electrostatic solver, as this may lead to out-of-bounds) if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(rho->nComp() >= 2, + "Cannot deposit charge in rho component 1: only component 0 is allocated!"); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 142e973d1b2..7199d420173 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -883,6 +883,12 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const long offset, const long np_to_deposit, const int thread_num, const int lev, const int depos_lev) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + rho->nComp() >= icomp - 1, + "Cannot deposit charge in rho component icomp=" + std::to_string(icomp) + + ": not enough components allocated (" + std::to_string(rho->nComp()) + "!" + ); + if (WarpX::do_shared_mem_charge_deposition) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || @@ -1208,6 +1214,12 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, const bool apply_boundary_and_scale_volume, const int icomp) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + rho->nComp() >= icomp - 1, + "Cannot deposit charge in rho component icomp=" + std::to_string(icomp) + + ": not enough components allocated (" + std::to_string(rho->nComp()) + "!" + ); + // Reset the rho array if reset is True int const nc = WarpX::ncomps; if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } From 8344f2ccaceadcdb36866a2143ae007a8d2bd297 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 23 Apr 2024 13:25:02 -0700 Subject: [PATCH 079/121] AMReX: Weekly Update (#4881) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 06eb90b4395..abd7e54c013 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - + cd ../amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index a54553689ad..048efbc06be 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 +branch = b752027c1aebdfb4be339b1e30932b4108286a7a [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f35a7676ab8..1a157d7e022 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 +branch = b752027c1aebdfb4be339b1e30932b4108286a7a [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9d21519ce71..022939dcbee 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "7813bb7d6bc6cea658edac1f2bde046c7d174168" +set(WarpX_amrex_branch "b752027c1aebdfb4be339b1e30932b4108286a7a" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 23602682ff5..9bfc3ba5fbf 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - +cd amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 899345711fa27eaf0c7623bfcaeb0b2d797c4922 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:38:56 -0700 Subject: [PATCH 080/121] Reorder particle boundary handling and call to `Redistribute` (#4852) * Reorder particle boundary handling and call to `Redistribute` * split boundary buffer gathering into separate functions for domain and embedded boundaries * rename `scrapeParticles` -> `scrapeParticlesAtEB` --- Source/EmbeddedBoundary/ParticleScraper.H | 12 ++++++------ Source/Evolve/WarpXEvolve.cpp | 15 ++++++++------- Source/Particles/MultiParticleContainer.H | 4 +++- Source/Particles/MultiParticleContainer.cpp | 12 ++++++++++-- Source/Particles/ParticleBoundaryBuffer.H | 6 ++++-- Source/Particles/ParticleBoundaryBuffer.cpp | 14 +++++++++++--- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/Particles/WarpXParticleContainer.cpp | 2 +- 8 files changed, 45 insertions(+), 24 deletions(-) diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index d058de7e48f..a382a454ec6 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -65,9 +65,9 @@ */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) { - scrapeParticles(pc, distance_to_eb, lev, lev, std::forward(f)); + scrapeParticlesAtEB(pc, distance_to_eb, lev, lev, std::forward(f)); } /** @@ -108,9 +108,9 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, F&& f) +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, F&& f) { - scrapeParticles(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); + scrapeParticlesAtEB(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); } /** @@ -153,10 +153,10 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev_min, int lev_max, F&& f) { - BL_PROFILE("scrapeParticles"); + BL_PROFILE("scrapeParticlesAtEB"); for (int lev = lev_min; lev <= lev_max; ++lev) { diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index a8aa89355ed..51fcb5c262c 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -489,13 +489,7 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num mypc->ContinuousFluxInjection(cur_time, dt[0]); mypc->ApplyBoundaryConditions(); - - // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); -#endif - - m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + m_particle_boundary_buffer->gatherParticlesFromDomainBoundaries(*mypc); // Non-Maxwell solver: particles can move by an arbitrary number of cells if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || @@ -523,6 +517,13 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num } } + // interact the particles with EB walls (if present) +#ifdef AMREX_USE_EB + mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); + m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + mypc->deleteInvalidParticles(); +#endif + if (sort_intervals.contains(step+1)) { if (verbose) { amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 0421febc475..f007d5088e3 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -213,6 +213,8 @@ public: void defineAllParticleTiles (); + void deleteInvalidParticles (); + void RedistributeLocal (int num_ghost); /** Apply BC. For now, just discard particles outside the domain, regardless @@ -294,7 +296,7 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - void ScrapeParticles (const amrex::Vector& distance_to_eb); + void ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb); std::string m_B_ext_particle_s = "none"; std::string m_E_ext_particle_s = "none"; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 97c51c6350b..dda32467925 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -632,6 +632,14 @@ MultiParticleContainer::defineAllParticleTiles () } } +void +MultiParticleContainer::deleteInvalidParticles () +{ + for (auto& pc : allcontainers) { + pc->deleteInvalidParticles(); + } +} + void MultiParticleContainer::RedistributeLocal (const int num_ghost) { @@ -945,11 +953,11 @@ void MultiParticleContainer::CheckIonizationProductSpecies() } } -void MultiParticleContainer::ScrapeParticles (const amrex::Vector& distance_to_eb) +void MultiParticleContainer::ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb) { #ifdef AMREX_USE_EB for (auto& pc : allcontainers) { - scrapeParticles(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); } #else amrex::ignore_unused(distance_to_eb); diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 20bc63f7a19..895a3810852 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -39,8 +39,10 @@ public: const std::vector& getSpeciesNames() const; - void gatherParticles (MultiParticleContainer& mypc, - const amrex::Vector& distance_to_eb); + void gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc); + void gatherParticlesFromEmbeddedBoundaries ( + MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb + ); void redistribute (); void clearParticles (); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 25b77ddc496..c8c683f0abf 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -354,8 +354,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { } } -void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, - const amrex::Vector& distance_to_eb) +void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc) { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); @@ -439,10 +438,19 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } } } +} +void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( + MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb) +{ #ifdef AMREX_USE_EB WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); + using PIter = amrex::ParConstIterSoA; + const auto& warpx_instance = WarpX::GetInstance(); + const amrex::Geometry& geom = warpx_instance.Geom(0); + auto plo = geom.ProbLoArray(); + auto& buffer = m_particle_containers[m_particle_containers.size()-1]; for (int i = 0; i < numSpecies(); ++i) { @@ -526,7 +534,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } } #else - amrex::ignore_unused(distance_to_eb); + amrex::ignore_unused(mypc, distance_to_eb); #endif } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e979afa9f77..f8afb1d39af 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1476,7 +1476,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); #endif // The function that calls this is responsible for redistributing particles. @@ -1973,7 +1973,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); #endif // Redistribute the new particles that were added to the temporary container. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 7199d420173..8692dff3302 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -301,7 +301,7 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); deleteInvalidParticles(); #endif } From ae5085533c9075fe0e650b12b10194a9f0c2c9e6 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:00:23 -0700 Subject: [PATCH 081/121] remove offending calls to pip install in macOS build (#4887) --- .github/workflows/macos.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f34f9f3534d..21d5ae04faf 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -36,9 +36,6 @@ jobs: brew tap openpmd/openpmd brew install openpmd-api - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade virtualenv - python3 -m venv py-venv source py-venv/bin/activate python3 -m pip install --upgrade pip From cbe43551bb1f212eb42aff12db94643801e28678 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 02:00:58 +0200 Subject: [PATCH 082/121] make sure that include guards start with WARPX_ (#4876) --- Source/BoundaryConditions/PML_current.H | 6 +++--- Source/Diagnostics/BTD_Plotfile_Header_Impl.H | 6 +++--- Source/EmbeddedBoundary/DistanceToEB.H | 6 +++--- Source/EmbeddedBoundary/ParticleBoundaryProcess.H | 6 +++--- Source/EmbeddedBoundary/ParticleScraper.H | 6 +++--- Source/FieldSolver/ElectrostaticSolver.H | 6 +++--- .../FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H | 6 +++--- Source/FieldSolver/WarpXPushFieldsEM_K.H | 6 +++--- Source/FieldSolver/WarpX_QED_K.H | 6 +++--- Source/Fluids/MultiFluidContainer_fwd.H | 1 + Source/Fluids/WarpXFluidContainer_fwd.H | 1 + Source/Initialization/ExternalField.H | 6 +++--- Source/Initialization/ExternalField_fwd.H | 6 +++--- Source/Initialization/GetTemperature.H | 6 +++--- Source/Initialization/GetVelocity.H | 6 +++--- Source/Initialization/InjectorDensity.H | 6 +++--- Source/Initialization/InjectorFlux.H | 6 +++--- Source/Initialization/InjectorMomentum.H | 6 +++--- Source/Initialization/InjectorPosition.H | 6 +++--- Source/Initialization/PlasmaInjector.H | 6 +++--- Source/Initialization/SampleGaussianFluxDistribution.H | 6 +++--- Source/Initialization/TemperatureProperties.H | 6 +++--- Source/Initialization/VelocityProperties.H | 6 +++--- Source/Initialization/WarpXAMReXInit.H | 2 +- Source/Parallelization/GuardCellManager.H | 6 +++--- Source/Particles/Algorithms/KineticEnergy.H | 6 +++--- .../Collision/BinaryCollision/BinaryCollisionUtils.H | 6 +++--- .../BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H | 6 +++--- .../Collision/BinaryCollision/DSMC/CollisionFilterFunc.H | 6 +++--- Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H | 6 +++--- .../Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H | 6 +++--- .../NuclearFusion/BoschHaleFusionCrossSection.H | 6 +++--- .../BinaryCollision/NuclearFusion/NuclearFusionFunc.H | 6 +++--- .../NuclearFusion/ProtonBoronFusionCrossSection.H | 6 +++--- .../NuclearFusion/ProtonBoronFusionInitializeMomentum.H | 6 +++--- .../NuclearFusion/SingleNuclearFusionEvent.H | 6 +++--- .../NuclearFusion/TwoProductFusionInitializeMomentum.H | 6 +++--- .../BinaryCollision/NuclearFusion/TwoProductFusionUtil.H | 6 +++--- .../Collision/BinaryCollision/ParticleCreationFunc.H | 6 +++--- Source/Particles/Deposition/ChargeDeposition.H | 6 +++--- Source/Particles/Deposition/CurrentDeposition.H | 6 +++--- Source/Particles/Deposition/SharedDepositionUtils.H | 6 +++--- Source/Particles/ElementaryProcess/Ionization.H | 6 +++--- Source/Particles/ElementaryProcess/QEDPairGeneration.H | 6 +++--- Source/Particles/ElementaryProcess/QEDPhotonEmission.H | 6 +++--- Source/Particles/ElementaryProcess/QEDSchwingerProcess.H | 6 +++--- Source/Particles/Filter/FilterFunctors.H | 6 +++--- Source/Particles/Gather/FieldGather.H | 6 +++--- Source/Particles/Gather/ScaleFields.H | 2 +- Source/Particles/NamedComponentParticleContainer.H | 6 +++--- Source/Particles/ParticleBoundaries.H | 6 +++--- Source/Particles/ParticleBoundaries_K.H | 6 +++--- Source/Particles/ParticleBoundaryBuffer.H | 6 +++--- Source/Particles/ParticleCreation/DefaultInitialization.H | 6 +++--- Source/Particles/ParticleCreation/FilterCopyTransform.H | 6 +++--- .../ParticleCreation/FilterCreateTransformFromFAB.H | 6 +++--- Source/Particles/ParticleCreation/SmartCopy.H | 6 +++--- Source/Particles/ParticleCreation/SmartCreate.H | 6 +++--- Source/Particles/ParticleCreation/SmartUtils.H | 6 +++--- Source/Particles/ParticleIO.H | 6 +++--- Source/Particles/PinnedMemoryParticleContainer.H | 6 +++--- Source/Particles/ShapeFactors.H | 6 +++--- Source/Utils/WarpXAlgorithmSelection.H | 6 +++--- 63 files changed, 181 insertions(+), 179 deletions(-) diff --git a/Source/BoundaryConditions/PML_current.H b/Source/BoundaryConditions/PML_current.H index 88e94de1e74..9361ce13992 100644 --- a/Source/BoundaryConditions/PML_current.H +++ b/Source/BoundaryConditions/PML_current.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PML_CURRENT_H_ -#define PML_CURRENT_H_ +#ifndef WARPX_PML_CURRENT_H_ +#define WARPX_PML_CURRENT_H_ #include "BoundaryConditions/PMLComponent.H" @@ -143,4 +143,4 @@ void damp_jz_pml (int j, int k, int l, #endif } -#endif +#endif //WARPX_PML_CURRENT_H_ diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 9d3250fa52f..7a1f8275bca 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef BTD_PLOTFILE_HEADER_IMPL_H -#define BTD_PLOTFILE_HEADER_IMPL_H +#ifndef WARPX_BTD_PLOTFILE_HEADER_IMPL_H +#define WARPX_BTD_PLOTFILE_HEADER_IMPL_H #include #include @@ -342,4 +342,4 @@ public: std::string m_Header_path; }; -#endif +#endif //WARPX_BTD_PLOTFILE_HEADER_IMPL_H diff --git a/Source/EmbeddedBoundary/DistanceToEB.H b/Source/EmbeddedBoundary/DistanceToEB.H index b46def938db..7ee47c1172c 100644 --- a/Source/EmbeddedBoundary/DistanceToEB.H +++ b/Source/EmbeddedBoundary/DistanceToEB.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef DISTANCETOEB_H_ -#define DISTANCETOEB_H_ +#ifndef WARPX_DISTANCETOEB_H_ +#define WARPX_DISTANCETOEB_H_ #include "Utils/TextMsg.H" @@ -129,4 +129,4 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP } #endif // AMREX_USE_EB -#endif // DISTANCETOEB_H_ +#endif // WARPX_DISTANCETOEB_H_ diff --git a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H index 3099f5df43d..345cafe7b4f 100644 --- a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H +++ b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARYPROCESS_H_ -#define PARTICLEBOUNDARYPROCESS_H_ +#ifndef WARPX_PARTICLEBOUNDARYPROCESS_H_ +#define WARPX_PARTICLEBOUNDARYPROCESS_H_ #include #include @@ -36,4 +36,4 @@ struct Absorb { }; } -#endif +#endif //WARPX_PARTICLEBOUNDARYPROCESS_H_ diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index a382a454ec6..ea4d0bc5c0e 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLESCRAPER_H_ -#define PARTICLESCRAPER_H_ +#ifndef WARPX_PARTICLESCRAPER_H_ +#define WARPX_PARTICLESCRAPER_H_ #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -216,4 +216,4 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan } } -#endif +#endif //WARPX_PARTICLESCRAPER_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 0a2abe2d6b2..c5c2ce8b6eb 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef ELECTROSTATICSOLVER_H_ -#define ELECTROSTATICSOLVER_H_ +#ifndef WARPX_ELECTROSTATICSOLVER_H_ +#define WARPX_ELECTROSTATICSOLVER_H_ #include #include @@ -122,4 +122,4 @@ namespace ElectrostaticSolver { }; } // namespace ElectrostaticSolver -#endif // ELECTROSTATICSOLVER_H_ +#endif // WARPX_ELECTROSTATICSOLVER_H_ diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H index f84a8f963fc..a8bbc954e29 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef MAGNETOSTATICSOLVER_H_ -#define MAGNETOSTATICSOLVER_H_ +#ifndef WARPX_MAGNETOSTATICSOLVER_H_ +#define WARPX_MAGNETOSTATICSOLVER_H_ #include #include @@ -54,4 +54,4 @@ namespace MagnetostaticSolver { }; } // namespace MagnetostaticSolver -#endif //MAGNETOSTATICSOLVER_H_ +#endif //WARPX_MAGNETOSTATICSOLVER_H_ diff --git a/Source/FieldSolver/WarpXPushFieldsEM_K.H b/Source/FieldSolver/WarpXPushFieldsEM_K.H index d6e7e353a54..4ff7b32b517 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM_K.H +++ b/Source/FieldSolver/WarpXPushFieldsEM_K.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef WarpXPushFieldsEM_K_h -#define WarpXPushFieldsEM_K_h +#ifndef WARPX_PushFieldsEM_K_h +#define WARPX_PushFieldsEM_K_h #include "Utils/WarpXConst.H" @@ -117,4 +117,4 @@ void damp_field_in_guards( } } -#endif +#endif //WARPX_PushFieldsEM_K_h diff --git a/Source/FieldSolver/WarpX_QED_K.H b/Source/FieldSolver/WarpX_QED_K.H index 6c86bba67b6..eccb8d6e72d 100644 --- a/Source/FieldSolver/WarpX_QED_K.H +++ b/Source/FieldSolver/WarpX_QED_K.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef WarpX_QED_K_h -#define WarpX_QED_K_h +#ifndef WARPX_QED_K_h +#define WARPX_QED_K_h #include "Utils/WarpXConst.H" @@ -359,4 +359,4 @@ const amrex::Real dyi = 1._rt/dy; } -#endif +#endif //WARPX_QED_K_h diff --git a/Source/Fluids/MultiFluidContainer_fwd.H b/Source/Fluids/MultiFluidContainer_fwd.H index a1a55c3b450..01edf7f728c 100644 --- a/Source/Fluids/MultiFluidContainer_fwd.H +++ b/Source/Fluids/MultiFluidContainer_fwd.H @@ -6,6 +6,7 @@ */ #ifndef WARPX_MultiFluidContainer_fwd_H_ +#define WARPX_MultiFluidContainer_fwd_H_ class MultiFluidContainer; diff --git a/Source/Fluids/WarpXFluidContainer_fwd.H b/Source/Fluids/WarpXFluidContainer_fwd.H index 77a6bc0c991..86fbff98fd0 100644 --- a/Source/Fluids/WarpXFluidContainer_fwd.H +++ b/Source/Fluids/WarpXFluidContainer_fwd.H @@ -6,6 +6,7 @@ */ #ifndef WARPX_WarpXFluidContainer_fwd_H_ +#define WARPX_WarpXFluidContainer_fwd_H_ class WarpXFluidContainer; diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H index 7b64339a042..b32e361bff4 100644 --- a/Source/Initialization/ExternalField.H +++ b/Source/Initialization/ExternalField.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef EXTERNAL_FIELD_H_ -#define EXTERNAL_FIELD_H_ +#ifndef WARPX_EXTERNAL_FIELD_H_ +#define WARPX_EXTERNAL_FIELD_H_ #include "ExternalField_fwd.H" @@ -66,4 +66,4 @@ struct ExternalFieldParams std::string external_fields_path; }; -#endif //EXTERNAL_FIELD_H_ +#endif //WARPX_EXTERNAL_FIELD_H_ diff --git a/Source/Initialization/ExternalField_fwd.H b/Source/Initialization/ExternalField_fwd.H index f7a8547ecce..a2fd6471222 100644 --- a/Source/Initialization/ExternalField_fwd.H +++ b/Source/Initialization/ExternalField_fwd.H @@ -5,9 +5,9 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef EXTERNAL_FIELD_FWD_H_ -#define EXTERNAL_FIELD_FWD_H_ +#ifndef WARPX_EXTERNAL_FIELD_FWD_H_ +#define WARPX_EXTERNAL_FIELD_FWD_H_ struct ExternalFieldParams; -#endif //EXTERNAL_FIELD_FWD_H_ +#endif //WARPX_EXTERNAL_FIELD_FWD_H_ diff --git a/Source/Initialization/GetTemperature.H b/Source/Initialization/GetTemperature.H index 4437928697c..20760863fae 100644 --- a/Source/Initialization/GetTemperature.H +++ b/Source/Initialization/GetTemperature.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef GET_TEMPERATURE_H_ -#define GET_TEMPERATURE_H_ +#ifndef WARPX_GET_TEMPERATURE_H_ +#define WARPX_GET_TEMPERATURE_H_ #include "TemperatureProperties.H" @@ -69,4 +69,4 @@ struct GetTemperature } } }; -#endif +#endif //WARPX_GET_TEMPERATURE_H_ diff --git a/Source/Initialization/GetVelocity.H b/Source/Initialization/GetVelocity.H index 3123708bf06..c8535b8af8e 100644 --- a/Source/Initialization/GetVelocity.H +++ b/Source/Initialization/GetVelocity.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef GET_VELOCITY_H_ -#define GET_VELOCITY_H_ +#ifndef WARPX_GET_VELOCITY_H_ +#define WARPX_GET_VELOCITY_H_ #include "VelocityProperties.H" @@ -86,4 +86,4 @@ struct GetVelocity return m_dir; } }; -#endif +#endif //WARPX_GET_VELOCITY_H_ diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H index 3848a42f4ee..465627f874a 100644 --- a/Source/Initialization/InjectorDensity.H +++ b/Source/Initialization/InjectorDensity.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_DENSITY_H_ -#define INJECTOR_DENSITY_H_ +#ifndef WARPX_INJECTOR_DENSITY_H_ +#define WARPX_INJECTOR_DENSITY_H_ #include "Utils/WarpXConst.H" @@ -218,4 +218,4 @@ struct InjectorDensityDeleter { } }; -#endif +#endif //WARPX_INJECTOR_DENSITY_H_ diff --git a/Source/Initialization/InjectorFlux.H b/Source/Initialization/InjectorFlux.H index e889c8b8966..6d608822c31 100644 --- a/Source/Initialization/InjectorFlux.H +++ b/Source/Initialization/InjectorFlux.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_FLUX_H_ -#define INJECTOR_FLUX_H_ +#ifndef WARPX_INJECTOR_FLUX_H_ +#define WARPX_INJECTOR_FLUX_H_ #include "Utils/WarpXConst.H" @@ -149,4 +149,4 @@ struct InjectorFluxDeleter { } }; -#endif +#endif //WARPX_INJECTOR_FLUX_H_ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 2b8b9065a36..52fedcd779f 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_MOMENTUM_H_ -#define INJECTOR_MOMENTUM_H_ +#ifndef WARPX_INJECTOR_MOMENTUM_H_ +#define WARPX_INJECTOR_MOMENTUM_H_ #include "GetTemperature.H" #include "GetVelocity.H" @@ -771,4 +771,4 @@ struct InjectorMomentumDeleter { } }; -#endif +#endif //WARPX_INJECTOR_MOMENTUM_H_ diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 7168ac93ebb..1b3be9e28a7 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_POSITION_H_ -#define INJECTOR_POSITION_H_ +#ifndef WARPX_INJECTOR_POSITION_H_ +#define WARPX_INJECTOR_POSITION_H_ #include "InjectorPosition_fwd.H" @@ -255,4 +255,4 @@ private: amrex::Real zmin, zmax; }; -#endif +#endif //WARPX_INJECTOR_POSITION_H_ diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 2651aad455f..b9fe2323290 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PLASMA_INJECTOR_H_ -#define PLASMA_INJECTOR_H_ +#ifndef WARPX_PLASMA_INJECTOR_H_ +#define WARPX_PLASMA_INJECTOR_H_ #include "InjectorDensity.H" #include "InjectorFlux.H" @@ -202,4 +202,4 @@ protected: void parseFlux (amrex::ParmParse const& pp_species); }; -#endif +#endif //WARPX_PLASMA_INJECTOR_H_ diff --git a/Source/Initialization/SampleGaussianFluxDistribution.H b/Source/Initialization/SampleGaussianFluxDistribution.H index 407c9df632a..65cad141e8f 100644 --- a/Source/Initialization/SampleGaussianFluxDistribution.H +++ b/Source/Initialization/SampleGaussianFluxDistribution.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H -#define SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#ifndef WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#define WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H #include @@ -78,4 +78,4 @@ namespace { } } -#endif +#endif //WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H diff --git a/Source/Initialization/TemperatureProperties.H b/Source/Initialization/TemperatureProperties.H index 4d30b67c851..78a967a32fa 100644 --- a/Source/Initialization/TemperatureProperties.H +++ b/Source/Initialization/TemperatureProperties.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TEMPERATURE_PROPERTIES_H_ -#define TEMPERATURE_PROPERTIES_H_ +#ifndef WARPX_TEMPERATURE_PROPERTIES_H_ +#define WARPX_TEMPERATURE_PROPERTIES_H_ #include #include @@ -43,4 +43,4 @@ struct TemperatureProperties std::unique_ptr m_ptr_temperature_parser; }; -#endif +#endif //WARPX_TEMPERATURE_PROPERTIES_H_ diff --git a/Source/Initialization/VelocityProperties.H b/Source/Initialization/VelocityProperties.H index 453d97725e2..87730acbc3a 100644 --- a/Source/Initialization/VelocityProperties.H +++ b/Source/Initialization/VelocityProperties.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef VELOCITY_PROPERTIES_H_ -#define VELOCITY_PROPERTIES_H_ +#ifndef WARPX_VELOCITY_PROPERTIES_H_ +#define WARPX_VELOCITY_PROPERTIES_H_ #include #include @@ -50,4 +50,4 @@ struct VelocityProperties std::unique_ptr m_ptr_velocity_parser; }; -#endif +#endif //WARPX_VELOCITY_PROPERTIES_H_ diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 613250b5c2f..ab65fa2687c 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -33,4 +33,4 @@ namespace warpx::initialization ); } -#endif +#endif //WARPX_AMREX_INIT_H_ diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 5c530441fc9..341db01bef6 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef GUARDCELLMANAGER_H_ -#define GUARDCELLMANAGER_H_ +#ifndef WARPX_GUARDCELLMANAGER_H_ +#define WARPX_GUARDCELLMANAGER_H_ #include #include @@ -108,4 +108,4 @@ public: amrex::IntVect ng_depos_rho = amrex::IntVect::TheZeroVector(); }; -#endif // GUARDCELLMANAGER_H_ +#endif // WARPX_GUARDCELLMANAGER_H_ diff --git a/Source/Particles/Algorithms/KineticEnergy.H b/Source/Particles/Algorithms/KineticEnergy.H index 87da83f9d8d..f097b6436e1 100644 --- a/Source/Particles/Algorithms/KineticEnergy.H +++ b/Source/Particles/Algorithms/KineticEnergy.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLES_KINETIC_ENERGY_H_ -#define PARTICLES_KINETIC_ENERGY_H_ +#ifndef WARPX_PARTICLES_KINETIC_ENERGY_H_ +#define WARPX_PARTICLES_KINETIC_ENERGY_H_ #include "Utils/WarpXConst.H" @@ -68,4 +68,4 @@ namespace Algorithms{ } -#endif // PARTICLES_ALGORITHMS_H_ +#endif // WARPX_PARTICLES_KINETIC_ENERGY_H_ diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index fdc88a275a0..091bd41955f 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef BINARY_COLLISION_UTILS_H_ -#define BINARY_COLLISION_UTILS_H_ +#ifndef WARPX_BINARY_COLLISION_UTILS_H_ +#define WARPX_BINARY_COLLISION_UTILS_H_ #include @@ -128,4 +128,4 @@ namespace BinaryCollisionUtils{ } } -#endif // BINARY_COLLISION_UTILS_H_ +#endif // WARPX_BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index 79fe1d42b7d..26782c2e42a 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PAIRWISE_COULOMB_COLLISION_FUNC_H_ -#define PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#ifndef WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#define WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ #include "ElasticCollisionPerez.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -117,4 +117,4 @@ private: Executor m_exe; }; -#endif // PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#endif // WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index bc11e5e4a99..46b228b049e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef COLLISION_FILTER_FUNC_H_ -#define COLLISION_FILTER_FUNC_H_ +#ifndef WARPX_COLLISION_FILTER_FUNC_H_ +#define WARPX_COLLISION_FILTER_FUNC_H_ #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" @@ -109,4 +109,4 @@ void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleRe } } -#endif // COLLISION_FILTER_FUNC_H_ +#endif // WARPX_COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 3c3849580d1..f28f3fb984c 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -7,8 +7,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef DSMC_FUNC_H_ -#define DSMC_FUNC_H_ +#ifndef WARPX_DSMC_FUNC_H_ +#define WARPX_DSMC_FUNC_H_ #include "CollisionFilterFunc.H" @@ -201,4 +201,4 @@ private: Executor m_exe; }; -#endif // DSMC_FUNC_H_ +#endif // WARPX_DSMC_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index a44ab90c4cc..86a2c70374e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SPLIT_AND_SCATTER_FUNC_H_ -#define SPLIT_AND_SCATTER_FUNC_H_ +#ifndef WARPX_SPLIT_AND_SCATTER_FUNC_H_ +#define WARPX_SPLIT_AND_SCATTER_FUNC_H_ #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" @@ -291,4 +291,4 @@ private: amrex::Gpu::HostVector m_num_products_host; CollisionType m_collision_type; }; -#endif // SPLIT_AND_SCATTER_FUNC_H_ +#endif // WARPX_SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H index 718d386d5ba..e408600480e 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef BOSCH_HALE_FUSION_CROSS_SECTION_H -#define BOSCH_HALE_FUSION_CROSS_SECTION_H +#ifndef WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H +#define WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Utils/WarpXConst.H" @@ -110,4 +110,4 @@ amrex::ParticleReal BoschHaleFusionCrossSection ( return millibarn_to_sqm * astrophysical_factor/E_keV * std::exp(-B_G/std::sqrt(E_keV)); } -#endif // BOSCH_HALE_FUSION_CROSS_SECTION_H +#endif // WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index a7dbc67b862..b6d6fe171de 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef NUCLEAR_FUSION_FUNC_H_ -#define NUCLEAR_FUSION_FUNC_H_ +#ifndef WARPX_NUCLEAR_FUSION_FUNC_H_ +#define WARPX_NUCLEAR_FUSION_FUNC_H_ #include "SingleNuclearFusionEvent.H" @@ -255,4 +255,4 @@ private: Executor m_exe; }; -#endif // NUCLEAR_FUSION_FUNC_H_ +#endif // WARPX_NUCLEAR_FUSION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index ec9a1989189..a5549f30b6f 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PROTON_BORON_FUSION_CROSS_SECTION_H -#define PROTON_BORON_FUSION_CROSS_SECTION_H +#ifndef WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H +#define WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H #include "Utils/WarpXConst.H" @@ -154,4 +154,4 @@ amrex::ParticleReal ProtonBoronFusionCrossSection (const amrex::ParticleReal& E_ return cross_section_b*barn_to_sqm; } -#endif // PROTON_BORON_FUSION_CROSS_SECTION_H +#endif // WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 0b51d6b4b61..375cc1e6d51 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H -#define PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#ifndef WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#define WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H #include "TwoProductFusionUtil.H" #include "Particles/WarpXParticleContainer.H" @@ -177,4 +177,4 @@ namespace { } -#endif // PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#endif // WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 07e0174438b..9bd4441eaa2 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SINGLE_NUCLEAR_FUSION_EVENT_H_ -#define SINGLE_NUCLEAR_FUSION_EVENT_H_ +#ifndef WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ +#define WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ #include "BoschHaleFusionCrossSection.H" #include "ProtonBoronFusionCrossSection.H" @@ -131,4 +131,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } -#endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ +#endif // WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index 52e9db8aa94..6ca87b9390d 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H -#define TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#ifndef WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#define WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H #include "TwoProductFusionUtil.H" #include "Particles/WarpXParticleContainer.H" @@ -96,4 +96,4 @@ namespace { } } -#endif // TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#endif // WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H index 09afe1e13a2..dbdef531a16 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TWO_PRODUCT_FUSION_UTIL_H -#define TWO_PRODUCT_FUSION_UTIL_H +#ifndef WARPX_TWO_PRODUCT_FUSION_UTIL_H +#define WARPX_TWO_PRODUCT_FUSION_UTIL_H #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -165,4 +165,4 @@ namespace { } } -#endif // TWO_PRODUCT_FUSION_UTIL_H +#endif // WARPX_TWO_PRODUCT_FUSION_UTIL_H diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 83312017e44..bf31076d975 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLE_CREATION_FUNC_H_ -#define PARTICLE_CREATION_FUNC_H_ +#ifndef WARPX_PARTICLE_CREATION_FUNC_H_ +#define WARPX_PARTICLE_CREATION_FUNC_H_ #include "BinaryCollisionUtils.H" @@ -338,4 +338,4 @@ public: } }; -#endif // PARTICLE_CREATION_FUNC_H_ +#endif // WARPX_PARTICLE_CREATION_FUNC_H_ diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 8ff712eefa2..2912b769c57 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef CHARGEDEPOSITION_H_ -#define CHARGEDEPOSITION_H_ +#ifndef WARPX_CHARGEDEPOSITION_H_ +#define WARPX_CHARGEDEPOSITION_H_ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" @@ -468,4 +468,4 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif } -#endif // CHARGEDEPOSITION_H_ +#endif // WARPX_CHARGEDEPOSITION_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index e722839c9c6..3dce7d28534 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef CURRENTDEPOSITION_H_ -#define CURRENTDEPOSITION_H_ +#ifndef WARPX_CURRENTDEPOSITION_H_ +#define WARPX_CURRENTDEPOSITION_H_ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" @@ -2378,4 +2378,4 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, #endif // #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) } -#endif // CURRENTDEPOSITION_H_ +#endif // WARPX_CURRENTDEPOSITION_H_ diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index e28835a57df..1e2294be3a2 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SHAREDDEPOSITIONUTILS_H_ -#define SHAREDDEPOSITIONUTILS_H_ +#ifndef WARPX_SHAREDDEPOSITIONUTILS_H_ +#define WARPX_SHAREDDEPOSITIONUTILS_H_ #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/ShapeFactors.H" @@ -281,4 +281,4 @@ void depositComponent (const GetParticlePosition& GetPosition, } #endif -#endif // SHAREDDEPOSITIONUTILS_H_ +#endif // WARPX_SHAREDDEPOSITIONUTILS_H_ diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 61aeb19aea1..cee7ec07eb5 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef IONIZATION_H_ -#define IONIZATION_H_ +#ifndef WARPX_IONIZATION_H_ +#define WARPX_IONIZATION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -171,4 +171,4 @@ struct IonizationTransformFunc } }; -#endif +#endif //WARPX_IONIZATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index fb723f0b79a..d550c5f03ee 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_PAIR_GENERATION_H_ -#define QED_PAIR_GENERATION_H_ +#ifndef WARPX_QED_PAIR_GENERATION_H_ +#define WARPX_QED_PAIR_GENERATION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -208,4 +208,4 @@ private: amrex::Dim3 m_lo; }; -#endif //QED_PAIR_GENERATION_H_ +#endif //WARPX_QED_PAIR_GENERATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 567b260d0e4..dd5f57a8f24 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_PHOTON_EMISSION_H_ -#define QED_PHOTON_EMISSION_H_ +#ifndef WARPX_QED_PHOTON_EMISSION_H_ +#define WARPX_QED_PHOTON_EMISSION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -267,4 +267,4 @@ void cleanLowEnergyPhotons( } -#endif //QED_PHOTON_EMISSION_H_ +#endif //WARPX_QED_PHOTON_EMISSION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H index bf4c566b3ba..32b58dc50dc 100644 --- a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H +++ b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_SCHWINGER_PROCESS_H_ -#define QED_SCHWINGER_PROCESS_H_ +#ifndef WARPX_QED_SCHWINGER_PROCESS_H_ +#define WARPX_QED_SCHWINGER_PROCESS_H_ #include "Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H" #include "Utils/TextMsg.H" @@ -95,4 +95,4 @@ struct SchwingerTransformFunc } }; -#endif // QED_SCHWINGER_PROCESS_H_ +#endif // WARPX_QED_SCHWINGER_PROCESS_H_ diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index d1206eb022c..982eeb0d23a 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FILTERFUNCTORS_H -#define FILTERFUNCTORS_H +#ifndef WARPX_FILTERFUNCTORS_H +#define WARPX_FILTERFUNCTORS_H #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/WarpXParticleContainer.H" @@ -176,4 +176,4 @@ private: const amrex::RealBox m_domain; }; -#endif // FILTERFUNCTORS_H +#endif // WARPX_FILTERFUNCTORS_H diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index e51cfa41c58..b5bd4376ba1 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FIELDGATHER_H_ -#define FIELDGATHER_H_ +#ifndef WARPX_FIELDGATHER_H_ +#define WARPX_FIELDGATHER_H_ #include "Particles/Gather/GetExternalFields.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -1852,4 +1852,4 @@ void doGatherShapeNImplicit ( } } -#endif // FIELDGATHER_H_ +#endif // WARPX_FIELDGATHER_H_ diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 4cddfa5a342..5731bc047f4 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -60,4 +60,4 @@ struct ScaleFields } }; -#endif +#endif //WARPX_PARTICLES_GATHER_SCALEFIELDS_H_ diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index 983d417d9c0..02f4c44314a 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef NamedComponentParticleContainer_H_ -#define NamedComponentParticleContainer_H_ +#ifndef WARPX_NamedComponentParticleContainer_H_ +#define WARPX_NamedComponentParticleContainer_H_ #include "Utils/TextMsg.H" @@ -219,4 +219,4 @@ protected: std::map particle_runtime_icomps; }; -#endif +#endif //WARPX_NamedComponentParticleContainer_H_ diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index c056436770d..a1b31f61cd2 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARIES_H_ -#define PARTICLEBOUNDARIES_H_ +#ifndef WARPX_PARTICLEBOUNDARIES_H_ +#define WARPX_PARTICLEBOUNDARIES_H_ #include "Utils/WarpXAlgorithmSelection.H" @@ -70,4 +70,4 @@ struct ParticleBoundaries ParticleBoundariesData data; }; -#endif /*PARTICLEBOUNDARIES_H_*/ +#endif /*WARPX_PARTICLEBOUNDARIES_H_*/ diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 1481dca7684..71b95aae95a 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARIES_K_H_ -#define PARTICLEBOUNDARIES_K_H_ +#ifndef WARPX_PARTICLEBOUNDARIES_K_H_ +#define WARPX_PARTICLEBOUNDARIES_K_H_ #include "ParticleBoundaries.H" #include "Initialization/SampleGaussianFluxDistribution.H" @@ -186,4 +186,4 @@ namespace ApplyParticleBoundaries { } } -#endif +#endif //WARPX_PARTICLEBOUNDARIES_K_H_ diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 895a3810852..d33834309ab 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARYBUFFER_H_ -#define PARTICLEBOUNDARYBUFFER_H_ +#ifndef WARPX_PARTICLEBOUNDARYBUFFER_H_ +#define WARPX_PARTICLEBOUNDARYBUFFER_H_ #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer.H" @@ -82,4 +82,4 @@ private: mutable std::vector m_species_names; }; -#endif /*PARTICLEBOUNDARYBUFFER_H_*/ +#endif /*WARPX_PARTICLEBOUNDARYBUFFER_H_*/ diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 3f0cbab79d5..88b23905481 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef DEFAULTINITIALIZATION_H_ -#define DEFAULTINITIALIZATION_H_ +#ifndef WARPX_DEFAULTINITIALIZATION_H_ +#define WARPX_DEFAULTINITIALIZATION_H_ #include #ifdef WARPX_QED @@ -280,4 +280,4 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, } -#endif +#endif //WARPX_DEFAULTINITIALIZATION_H_ diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 095ebbf6a85..4815a98ca31 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FILTER_COPY_TRANSFORM_H_ -#define FILTER_COPY_TRANSFORM_H_ +#ifndef WARPX_FILTER_COPY_TRANSFORM_H_ +#define WARPX_FILTER_COPY_TRANSFORM_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" @@ -360,4 +360,4 @@ Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi std::forward(transform)); } -#endif +#endif //WARPX_FILTER_COPY_TRANSFORM_H_ diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 2a4c9fccad0..1fe44bdfa7f 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ -#define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#ifndef WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#define WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" #include "WarpX.H" @@ -244,4 +244,4 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi std::forward(transform)); } -#endif // FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#endif // WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 6a6ceb3d290..e1d944e9c30 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SMART_COPY_H_ -#define SMART_COPY_H_ +#ifndef WARPX_SMART_COPY_H_ +#define WARPX_SMART_COPY_H_ #include "DefaultInitialization.H" #include "SmartUtils.H" @@ -163,4 +163,4 @@ public: [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; -#endif +#endif //WARPX_SMART_COPY_H_ diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index b4f25d5daad..fe4cb5929e0 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SMART_CREATE_H_ -#define SMART_CREATE_H_ +#ifndef WARPX_SMART_CREATE_H_ +#define WARPX_SMART_CREATE_H_ #include "DefaultInitialization.H" @@ -112,4 +112,4 @@ public: [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; -#endif //SMART_CREATE_H_ +#endif //WARPX_SMART_CREATE_H_ diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index dbac563ca28..652a3aecd17 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SMART_UTILS_H_ -#define SMART_UTILS_H_ +#ifndef WARPX_SMART_UTILS_H_ +#define WARPX_SMART_UTILS_H_ #include "DefaultInitialization.H" @@ -70,4 +70,4 @@ void setNewParticleIDs (PTile& ptile, amrex::Long old_size, amrex::Long num_adde }); } -#endif //SMART_UTILS_H_ +#endif //WARPX_SMART_UTILS_H_ diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index 022ab471362..d5fc68f4097 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEIO_H_ -#define PARTICLEIO_H_ +#ifndef WARPX_PARTICLEIO_H_ +#define WARPX_PARTICLEIO_H_ #include "Particles/WarpXParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -92,4 +92,4 @@ void storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, int electrostatic_solver_id, bool is_full_diagnostic ); -#endif /* PARTICLEIO_H_ */ +#endif /* WARPX_PARTICLEIO_H_ */ diff --git a/Source/Particles/PinnedMemoryParticleContainer.H b/Source/Particles/PinnedMemoryParticleContainer.H index eb8fe835c73..402c621eb9a 100644 --- a/Source/Particles/PinnedMemoryParticleContainer.H +++ b/Source/Particles/PinnedMemoryParticleContainer.H @@ -1,8 +1,8 @@ -#ifndef PinnedMemoryParticleContainer_H_ -#define PinnedMemoryParticleContainer_H_ +#ifndef WARPX_PinnedMemoryParticleContainer_H_ +#define WARPX_PinnedMemoryParticleContainer_H_ #include "NamedComponentParticleContainer.H" using PinnedMemoryParticleContainer = NamedComponentParticleContainer; -#endif +#endif //WARPX_PinnedMemoryParticleContainer_H_ diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 7c56da457ed..e711cce15ff 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SHAPEFACTORS_H_ -#define SHAPEFACTORS_H_ +#ifndef WARPX_SHAPEFACTORS_H_ +#define WARPX_SHAPEFACTORS_H_ #include "Utils/TextMsg.H" @@ -243,4 +243,4 @@ struct Compute_shape_factor_pair } }; -#endif // SHAPEFACTORS_H_ +#endif // WARPX_SHAPEFACTORS_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 9768b36cb84..c34516aedc5 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef UTILS_WARPXALGORITHMSELECTION_H_ -#define UTILS_WARPXALGORITHMSELECTION_H_ +#ifndef WARPX_UTILS_WARPXALGORITHMSELECTION_H_ +#define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include @@ -207,4 +207,4 @@ GetParticleBCTypeInteger( std::string BCType ); std::string GetFieldBCTypeString( FieldBoundaryType fb_type ); -#endif // UTILS_WARPXALGORITHMSELECTION_H_ +#endif // WARPX_UTILS_WARPXALGORITHMSELECTION_H_ From ec175e55fa06fbc7134f551b9042a42d90020094 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:17:26 -0700 Subject: [PATCH 083/121] Implement grid based merging algorithm (#4820) * initial implementation of the grid based merging routine * add compiler directives for different dimensions * avoid implicit capture of `this` in lambda * code clean-up * use `PIdx::x` for r-coordinate while 4667 is still being reviewed * fix clang-tidy errors * another clang-tidy fix * improve doc-strings * use iterative heap sort since "SYCL kernel cannot call a recursive function" * fix clang-tidy error * fix sorting bug; add documentation and CI test * add dummy CI benchmark file to get proper values * update benchmark values based on Azure results * rename `GridBasedMerging` -> `VelocityCoincidenceThinning` * use `Algorithms::KineticEnergy` * reorganize merging loop * update CI benchmark values * Relativistic correction in product particles' velocity calculation * update benchmark values after changing energy calculation * handle edge case with zero cluster momentum * call redistribute after particle resampling to remove invalid particles * use unsigned ints for indexing * Revert "use unsigned ints for indexing" This reverts commit abe027f3402cf0e7c51c4599a0a50b9e8a6b817a. * call `Redistribute` before merging * code clean-up * also check for `std::isnan` in edge case handling * add reference for grid based merging * check that cluster has total weight > 0 before merging * add defense against numerical error leading to nan * make resampling message more verbose * use `deleteInvalidParticles` instead of `Redistribute` * remove default values for merging parameters * remove doc-string default specifications in picmi.py * apply suggestions from code review * update benchmark values; avoid possible nans * add assert to prevent merging of photons * add `BackwardCompatibility` check to `LevelingThinning` --- Docs/source/refs.bib | 11 + Docs/source/usage/parameters.rst | 28 +- .../inputs_1d_velocity_coincidence_thinning | 46 +++ .../Tests/resampling/inputs_leveling_thinning | 2 +- Python/pywarpx/picmi.py | 31 +- ...esample_velocity_coincidence_thinning.json | 20 ++ Regression/WarpX-tests.ini | 17 ++ .../Particles/PhysicalParticleContainer.cpp | 14 +- Source/Particles/Resampling/CMakeLists.txt | 1 + .../Particles/Resampling/LevelingThinning.H | 6 + .../Particles/Resampling/LevelingThinning.cpp | 14 +- Source/Particles/Resampling/Make.package | 1 + Source/Particles/Resampling/Resampling.cpp | 5 + .../Resampling/VelocityCoincidenceThinning.H | 115 ++++++++ .../VelocityCoincidenceThinning.cpp | 278 ++++++++++++++++++ 15 files changed, 574 insertions(+), 15 deletions(-) create mode 100644 Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning create mode 100644 Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json create mode 100644 Source/Particles/Resampling/VelocityCoincidenceThinning.H create mode 100644 Source/Particles/Resampling/VelocityCoincidenceThinning.cpp diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 2de27908c5d..84b34b17351 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -430,3 +430,14 @@ @book{Wiedemann2015 doi = {10.1007/978-3-319-18317-6}, year = {2015} } + +@article{Vranic2015, + title = {Particle merging algorithm for PIC codes}, + author = {M. Vranic and T. Grismayer and J.L. Martins and R.A. Fonseca and L.O. Silva}, + journal = {Computer Physics Communications}, + volume = {191}, + pages = {65-73}, + year = {2015}, + issn = {0010-4655}, + doi = {https://doi.org/10.1016/j.cpc.2015.01.020}, +} diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 0a089358d21..9f88f2d8c6c 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1269,25 +1269,39 @@ Particle initialization * ``.do_resampling`` (`0` or `1`) optional (default `0`) If `1` resampling is performed for this species. This means that the number of macroparticles will be reduced at specific timesteps while preserving the distribution function as much as - possible (in particular the weight of the remaining particles will be increased on average). + possible (details depend on the chosen resampling algorithm). This can be useful in situations with continuous creation of particles (e.g. with ionization or with QED effects). At least one resampling trigger (see below) must be specified to actually perform resampling. * ``.resampling_algorithm`` (`string`) optional (default `leveling_thinning`) - The algorithm used for resampling. Currently there is only one option, which is already set by - default: + The algorithm used for resampling: * ``leveling_thinning`` This algorithm is defined in :cite:t:`param-MuravievCPC2021`. - It has two parameters: + It has one parameter: * ``.resampling_algorithm_target_ratio`` (`float`) optional (default `1.5`) This **roughly** corresponds to the ratio between the number of particles before and after resampling. - * ``.resampling_algorithm_min_ppc`` (`int`) optional (default `1`) - Resampling is not performed in cells with a number of macroparticles strictly smaller - than this parameter. + * ``velocity_coincidence_thinning``` The particles are sorted into phase space + cells and merged, similar to the approach described in :cite:t:`param-Vranic2015`. + It has three parameters: + + * ``.resampling_algorithm_delta_ur`` (`float`) + The width of momentum cells used in clustering particles, in m/s. + + * ``.resampling_algorithm_n_theta`` (`int`) + The number of cell divisions to use in the :math:`\theta` direction + when clustering the particle velocities. + + * ``.resampling_algorithm_n_phi`` (`int`) + The number of cell divisions to use in the :math:`\phi` direction + when clustering the particle velocities. + +* ``.resampling_min_ppc`` (`int`) optional (default `1`) + Resampling is not performed in cells with a number of macroparticles strictly smaller + than this parameter. * ``.resampling_trigger_intervals`` (`string`) optional (default `0`) Using the `Intervals parser`_ syntax, this string defines timesteps at which resampling is diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning new file mode 100644 index 00000000000..1610c8b1221 --- /dev/null +++ b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning @@ -0,0 +1,46 @@ +max_step = 4 +warpx.verbose = 1 +warpx.const_dt = 1e-10 +amr.n_cell = 256 +amr.max_grid_size = 64 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = 0 +geometry.prob_hi = 0.1 + +# Boundary condition and field solver +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic +algo.particle_shape = 1 +algo.maxwell_solver = none + +particles.species_names = hydrogen +hydrogen.mass = 1.67262192369e-27 +hydrogen.charge = 1.602176634e-19 +hydrogen.injection_style = nrandompercell +hydrogen.initialize_self_fields = 0 +hydrogen.do_not_push = 1 +hydrogen.do_resampling = 1 +hydrogen.resampling_min_ppc = 10 +hydrogen.resampling_trigger_intervals = 1::2 +hydrogen.resampling_algorithm = velocity_coincidence_thinning +hydrogen.resampling_algorithm_delta_ur = 100000000.0 +hydrogen.resampling_algorithm_n_phi = 3 +hydrogen.resampling_algorithm_n_theta = 120 +hydrogen.num_particles_per_cell = 2500 +hydrogen.momentum_distribution_type = gaussian +hydrogen.ux_m = 0.0 +hydrogen.uy_m = 0.0 +hydrogen.uz_m = 0.0 +hydrogen.ux_th = 0.000326 +hydrogen.uy_th = 0.000326 +hydrogen.uz_th = 0.000326 +hydrogen.profile = constant +hydrogen.density = 1e+19 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 4 +diag1.diag_type = Full diff --git a/Examples/Tests/resampling/inputs_leveling_thinning b/Examples/Tests/resampling/inputs_leveling_thinning index 9ed49ae358e..90e0d6802d6 100644 --- a/Examples/Tests/resampling/inputs_leveling_thinning +++ b/Examples/Tests/resampling/inputs_leveling_thinning @@ -57,7 +57,7 @@ resampled_part2.do_resampling = 1 resampled_part2.resampling_algorithm = leveling_thinning resampled_part2.resampling_algorithm_target_ratio = 1.3 # This should prevent actual resampling at timestep 7 -resampled_part2.resampling_algorithm_min_ppc = 80000 +resampled_part2.resampling_min_ppc = 80000 # This should trigger resampling at timestep 6 and 7 in this case. The rest is here to test the # intervals parser syntax. resampled_part2.resampling_trigger_intervals = 100 :120:3, 6:6:6 , 7 diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 4bf9d4b4163..2859d6fd468 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -132,12 +132,31 @@ class Species(picmistandard.PICMI_Species): warpx_do_resampling: bool, default=False Whether particles will be resampled + warpx_resampling_min_ppc: int, default=1 + Cells with fewer particles than this number will be + skipped during resampling. + warpx_resampling_trigger_intervals: bool, default=0 Timesteps at which to resample warpx_resampling_trigger_max_avg_ppc: int, default=infinity Resampling will be done when the average number of particles per cell exceeds this number + + warpx_resampling_algorithm: str, default="leveling_thinning" + Resampling algorithm to use. + + warpx_resampling_algorithm_delta_ur: float + Size of velocity window used for clustering particles during grid-based + merging. + + warpx_resampling_algorithm_n_theta: int + Number of bins to use in theta when clustering particle velocities + during grid-based merging. + + warpx_resampling_algorithm_n_phi: int + Number of bins to use in phi when clustering particle velocities + during grid-based merging. """ def init(self, kw): @@ -215,8 +234,13 @@ def init(self, kw): # Resampling settings self.do_resampling = kw.pop('warpx_do_resampling', None) + self.resampling_algorithm = kw.pop('warpx_resampling_algorithm', None) + self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) + self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) + self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) def species_initialize_inputs(self, layout, initialize_self_fields = False, @@ -255,8 +279,13 @@ def species_initialize_inputs(self, layout, do_not_gather = self.do_not_gather, random_theta = self.random_theta, do_resampling=self.do_resampling, + resampling_algorithm=self.resampling_algorithm, + resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, - resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc) + resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, + resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, + resampling_algorithm_n_phi=self.resampling_algorithm_n_phi) # add reflection models self.species.add_new_attr("reflection_model_xlo(E)", self.reflection_model_xlo) diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json new file mode 100644 index 00000000000..1f6f076f021 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 447073.7680171198, + "jy": 336566.4197455162, + "jz": 642997.1912520501 + }, + "hydrogen": { + "particle_momentum_x": 1.7562033925112653e-17, + "particle_momentum_y": 1.756681695333869e-17, + "particle_momentum_z": 2.0662018455459364e-17, + "particle_position_x": 6600.315303519964, + "particle_weight": 1.0000000000000003e+18 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 1a157d7e022..1f656960789 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4189,6 +4189,23 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py +[resample_velocity_coincidence_thinning] +buildDir = . +inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/analysis_default_regression.py + [restart] buildDir = . inputFile = Examples/Tests/restart/inputs diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index f8afb1d39af..c7f44903d9b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -3362,11 +3362,7 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose WARPX_PROFILE_VAR_START(blp_resample_actual); if (m_resampler.triggered(timestep, global_numparts)) { - if (verbose) { - amrex::Print() << Utils::TextMsg::Info( - "Resampling " + species_name + " at step " + std::to_string(timestep) - ); - } + Redistribute(); for (int lev = 0; lev <= maxLevel(); lev++) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) @@ -3374,6 +3370,14 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose m_resampler(pti, lev, this); } } + deleteInvalidParticles(); + if (verbose) { + amrex::Print() << Utils::TextMsg::Info( + "Resampled " + species_name + " at step " + std::to_string(timestep) + + ": macroparticle count decreased by " + + std::to_string(static_cast(global_numparts - TotalNumberOfParticles())) + ); + } } WARPX_PROFILE_VAR_STOP(blp_resample_actual); } diff --git a/Source/Particles/Resampling/CMakeLists.txt b/Source/Particles/Resampling/CMakeLists.txt index 13eb8a8c4b2..8fca0176b86 100644 --- a/Source/Particles/Resampling/CMakeLists.txt +++ b/Source/Particles/Resampling/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) Resampling.cpp ResamplingTrigger.cpp LevelingThinning.cpp + VelocityCoincidenceThinning.cpp ) endforeach() diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index 9e3e63fc8be..88e8aa79b4b 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -38,6 +38,12 @@ public: */ LevelingThinning (const std::string& species_name); + /** + * This function queries deprecated input parameters and aborts + * the run if one of them is specified. + */ + void BackwardCompatibility (const std::string& species_name ); + /** * \brief A method that performs leveling thinning for the considered species. * diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 71ebe735c78..09531d0e47d 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -47,11 +47,23 @@ LevelingThinning::LevelingThinning (const std::string& species_name) } utils::parser::queryWithParser( - pp_species_name, "resampling_algorithm_min_ppc", m_min_ppc); + pp_species_name, "resampling_min_ppc", m_min_ppc); + BackwardCompatibility(species_name); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_min_ppc >= 1, "Resampling min_ppc should be greater than or equal to 1"); } +void LevelingThinning::BackwardCompatibility (const std::string& species_name ) +{ + const amrex::ParmParse pp_species_name(species_name); + int backward_min_ppc; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !pp_species_name.query("resampling_algorithm_min_ppc", backward_min_ppc), + ".resampling_algorithm_min_ppc is no longer a valid option. " + "Please use the renamed option .resampling_min_ppc instead." + ); +} + void LevelingThinning::operator() (WarpXParIter& pti, const int lev, WarpXParticleContainer * const pc) const { diff --git a/Source/Particles/Resampling/Make.package b/Source/Particles/Resampling/Make.package index e70e3446c33..12b58dd9d47 100644 --- a/Source/Particles/Resampling/Make.package +++ b/Source/Particles/Resampling/Make.package @@ -1,5 +1,6 @@ CEXE_sources += Resampling.cpp CEXE_sources += ResamplingTrigger.cpp CEXE_sources += LevelingThinning.cpp +CEXE_sources += VelocityCoincidenceThinning.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Resampling/ diff --git a/Source/Particles/Resampling/Resampling.cpp b/Source/Particles/Resampling/Resampling.cpp index 11f8706dc33..15aba88dd3e 100644 --- a/Source/Particles/Resampling/Resampling.cpp +++ b/Source/Particles/Resampling/Resampling.cpp @@ -6,6 +6,7 @@ */ #include "Resampling.H" +#include "VelocityCoincidenceThinning.H" #include "LevelingThinning.H" #include "Utils/TextMsg.H" @@ -22,6 +23,10 @@ Resampling::Resampling (const std::string& species_name) { m_resampling_algorithm = std::make_unique(species_name); } + else if (resampling_algorithm_string == "velocity_coincidence_thinning") + { + m_resampling_algorithm = std::make_unique(species_name); + } else { WARPX_ABORT_WITH_MESSAGE("Unknown resampling algorithm."); } diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H new file mode 100644 index 00000000000..9462f4b35ed --- /dev/null +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -0,0 +1,115 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_VELOCITY_COINCIDENCE_THINNING_H_ +#define WARPX_VELOCITY_COINCIDENCE_THINNING_H_ + +#include "Resampling.H" + + +/** + * \brief This class implements a particle merging scheme wherein particles + * are clustered in phase space and particles in the same cluster is merged + * into two remaining particles. The scheme conserves linear momentum and + * kinetic energy within each cluster. + */ +class VelocityCoincidenceThinning: public ResamplingAlgorithm { +public: + + /** + * \brief Default constructor of the VelocityCoincidenceThinning class. + */ + VelocityCoincidenceThinning () = default; + + /** + * \brief Constructor of the VelocityCoincidenceThinning class + * + * @param[in] species_name the name of the resampled species + */ + VelocityCoincidenceThinning (const std::string& species_name); + + /** + * \brief A method that performs merging for the considered species. + * + * @param[in] pti WarpX particle iterator of the particles to resample. + * @param[in] lev the index of the refinement level. + * @param[in] pc a pointer to the particle container. + */ + void operator() (WarpXParIter& pti, int lev, WarpXParticleContainer* pc) const final; + + /** + * \brief This merging routine requires functionality to sort a GPU vector + * based on another GPU vector's values. The heap-sort functions below were + * obtained from https://www.geeksforgeeks.org/iterative-heap-sort/ and + * modified for the current purpose. It achieves the same as + * ``` + * std::sort( + * sorted_indices_data + cell_start, sorted_indices_data + cell_stop, + * [&momentum_bin_number_data](size_t i1, size_t i2) { + * return momentum_bin_number_data[i1] < momentum_bin_number_data[i2]; + * } + * ); + * ``` + * but with support for device execution. + */ + struct HeapSort { + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void swap(int &a, int &b) const + { + const auto temp = b; + b = a; + a = temp; + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator() (int index_array[], const int bin_array[], const int start, const int n) const + { + // sort index_array into a max heap structure + for (int i = 1; i < n; i++) + { + auto j = i; + // move child through heap if it is bigger than its parent + while (j > 0 && bin_array[index_array[j+start]] > bin_array[index_array[(j - 1)/2 + start]]) { + // swap child and parent until branch is properly ordered + swap(index_array[j+start], index_array[(j - 1)/2 + start]); + j = (j - 1) / 2; + } + } + + for (int i = n - 1; i > 0; i--) + { + // swap value of first (now the largest value) to the new end point + swap(index_array[start], index_array[i+start]); + + // remake the max heap + int j = 0, index; + while (j < i) { + index = 2 * j + 1; + + // if left child is smaller than right child, point index variable to right child + if (index + 1 < i && bin_array[index_array[index+start]] < bin_array[index_array[index+1+start]]) { + index++; + } + // if parent is smaller than child, swap parent with child having higher value + if (index < i && bin_array[index_array[j+start]] < bin_array[index_array[index+start]]) { + swap(index_array[j+start], index_array[index+start]); + } + j = index; + } + } + } + }; + +private: + int m_min_ppc = 1; + int m_ntheta; + int m_nphi; + amrex::ParticleReal m_delta_ur; +}; +#endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp new file mode 100644 index 00000000000..66176c48b4d --- /dev/null +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -0,0 +1,278 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "VelocityCoincidenceThinning.H" + +#include "Particles/Algorithms/KineticEnergy.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" + + +VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& species_name) +{ + using namespace amrex::literals; + + const amrex::ParmParse pp_species_name(species_name); + + utils::parser::queryWithParser( + pp_species_name, "resampling_min_ppc", m_min_ppc + ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_min_ppc >= 1, + "Resampling min_ppc should be greater than or equal to 1" + ); + + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_theta", m_ntheta + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_phi", m_nphi + ); +} + +void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, + WarpXParticleContainer * const pc) const +{ + using namespace amrex::literals; + + auto& ptile = pc->ParticlesAt(lev, pti); + auto& soa = ptile.GetStructOfArrays(); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) + auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); +#elif defined(WARPX_DIM_RZ) + auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); // rename to PIdx::r after PR #4667 +#endif +#if defined(WARPX_DIM_3D) + auto * const AMREX_RESTRICT y = soa.GetRealData(PIdx::y).data(); +#endif + auto * const AMREX_RESTRICT z = soa.GetRealData(PIdx::z).data(); + auto * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); + auto * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); + auto * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); + auto * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); + auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + + // Using this function means that we must loop over the cells in the ParallelFor. + auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); + + const auto n_cells = static_cast(bins.numBins()); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); + + const auto min_ppc = m_min_ppc; + + const auto mass = pc->getMass(); + + // check if species mass > 0 + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass > 0, + "VelocityCoincidenceThinning does not yet work for massless particles." + ); + + // create a GPU vector to hold the momentum cluster index for each particle + amrex::Gpu::DeviceVector momentum_bin_number(bins.numItems()); + auto* momentum_bin_number_data = momentum_bin_number.data(); + + // create a GPU vector to hold the index sorting for the momentum bins + amrex::Gpu::DeviceVector sorted_indices(bins.numItems()); + auto* sorted_indices_data = sorted_indices.data(); + + const auto Ntheta = m_ntheta; + const auto Nphi = m_nphi; + + const auto dr = m_delta_ur; + const auto dtheta = 2.0_prt * MathConst::pi / Ntheta; + const auto dphi = MathConst::pi / Nphi; + constexpr auto c2 = PhysConst::c * PhysConst::c; + + auto heapSort = HeapSort(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles that are in the cell `i_cell` are + // given by the `indices[cell_start:cell_stop]` + const auto cell_start = static_cast(cell_offsets[i_cell]); + const auto cell_stop = static_cast(cell_offsets[i_cell+1]); + const auto cell_numparts = cell_stop - cell_start; + + // do nothing for cells with less particles than min_ppc + // (this intentionally includes skipping empty cells, too) + if (cell_numparts < min_ppc) { + return; + } + + // Loop over particles and label them with the appropriate momentum bin + // number. Also assign initial ordering to the sorted_indices array. + for (int i = cell_start; i < cell_stop; ++i) + { + // get polar components of the velocity vector + auto u_mag = std::sqrt( + ux[indices[i]]*ux[indices[i]] + + uy[indices[i]]*uy[indices[i]] + + uz[indices[i]]*uz[indices[i]] + ); + auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; + auto u_phi = std::acos(uz[indices[i]]/u_mag); + + const auto ii = static_cast(u_theta / dtheta); + const auto jj = static_cast(u_phi / dphi); + const auto kk = static_cast(u_mag / dr); + + momentum_bin_number_data[i] = ii + jj * Ntheta + kk * Ntheta * Nphi; + sorted_indices_data[i] = i; + } + + // sort indices based on comparing values in momentum_bin_number + heapSort(sorted_indices_data, momentum_bin_number_data, cell_start, cell_numparts); + + // initialize variables used to hold cluster totals + int particles_in_bin = 0; + amrex::ParticleReal total_weight = 0._prt, total_energy = 0._prt; +#if !defined(WARPX_DIM_1D_Z) + amrex::ParticleReal cluster_x = 0._prt; +#endif +#if defined(WARPX_DIM_3D) + amrex::ParticleReal cluster_y = 0._prt; +#endif + amrex::ParticleReal cluster_z = 0._prt; + amrex::ParticleReal cluster_ux = 0._prt, cluster_uy = 0._prt, cluster_uz = 0._prt; + + // Finally, loop through the particles in the cell and merge + // ones in the same momentum bin + for (int i = cell_start; i < cell_stop; ++i) + { + particles_in_bin += 1; + const auto part_idx = indices[sorted_indices_data[i]]; + +#if !defined(WARPX_DIM_1D_Z) + cluster_x += w[part_idx]*x[part_idx]; +#endif +#if defined(WARPX_DIM_3D) + cluster_y += w[part_idx]*y[part_idx]; +#endif + cluster_z += w[part_idx]*z[part_idx]; + cluster_ux += w[part_idx]*ux[part_idx]; + cluster_uy += w[part_idx]*uy[part_idx]; + cluster_uz += w[part_idx]*uz[part_idx]; + total_weight += w[part_idx]; + total_energy += w[part_idx] * Algorithms::KineticEnergy( + ux[part_idx], uy[part_idx], uz[part_idx], mass + ); + + // check if this is the last particle in the current momentum bin + if ( + (i == cell_stop - 1) + || (momentum_bin_number_data[sorted_indices_data[i]] != momentum_bin_number_data[sorted_indices_data[i + 1]]) + ) { + // check if the bin has more than 2 particles in it + if ( particles_in_bin > 2 && total_weight > std::numeric_limits::min() ){ + // get average quantities for the bin +#if !defined(WARPX_DIM_1D_Z) + cluster_x /= total_weight; +#endif +#if defined(WARPX_DIM_3D) + cluster_y /= total_weight; +#endif + cluster_z /= total_weight; + cluster_ux /= total_weight; + cluster_uy /= total_weight; + cluster_uz /= total_weight; + + // perform merging of momentum bin particles + auto u_perp2 = cluster_ux*cluster_ux + cluster_uy*cluster_uy; + auto u_perp = std::sqrt(u_perp2); + auto cluster_u_mag2 = u_perp2 + cluster_uz*cluster_uz; + auto cluster_u_mag = std::sqrt(cluster_u_mag2); + + // calculate required velocity magnitude to achieve + // energy conservation + auto v_mag2 = total_energy / total_weight * ( + (total_energy / total_weight + 2._prt * mass * c2 ) + / (mass * mass * c2) + ); + auto v_perp = (v_mag2 > cluster_u_mag2) ? std::sqrt(v_mag2 - cluster_u_mag2) : 0_prt; + + // choose random angle for new velocity vector + auto phi = amrex::Random(engine) * MathConst::pi; + + // set new velocity components based on chosen phi + auto vx = v_perp * std::cos(phi); + auto vy = v_perp * std::sin(phi); + + // calculate rotation angles to parallel coord. frame + auto cos_theta = (cluster_u_mag > 0._prt) ? cluster_uz / cluster_u_mag : 0._prt; + auto sin_theta = (cluster_u_mag > 0._prt) ? u_perp / cluster_u_mag : 0._prt; + auto cos_phi = (u_perp > 0._prt) ? cluster_ux / u_perp : 0._prt; + auto sin_phi = (u_perp > 0._prt) ? cluster_uy / u_perp : 0._prt; + + // rotate new velocity vector to labframe + auto ux_new = ( + vx * cos_theta * cos_phi - vy * sin_phi + + cluster_u_mag * sin_theta * cos_phi + ); + auto uy_new = ( + vx * cos_theta * sin_phi + vy * cos_phi + + cluster_u_mag * sin_theta * sin_phi + ); + auto uz_new = -vx * sin_theta + cluster_u_mag * cos_theta; + + // set the last two particles' attributes according to + // the bin's aggregate values + const auto part_idx2 = indices[sorted_indices_data[i - 1]]; + + w[part_idx] = total_weight / 2._prt; + w[part_idx2] = total_weight / 2._prt; +#if !defined(WARPX_DIM_1D_Z) + x[part_idx] = cluster_x; + x[part_idx2] = cluster_x; +#endif +#if defined(WARPX_DIM_3D) + y[part_idx] = cluster_y; + y[part_idx2] = cluster_y; +#endif + z[part_idx] = cluster_z; + z[part_idx2] = cluster_z; + + ux[part_idx] = ux_new; + uy[part_idx] = uy_new; + uz[part_idx] = uz_new; + ux[part_idx2] = 2._prt * cluster_ux - ux_new; + uy[part_idx2] = 2._prt * cluster_uy - uy_new; + uz[part_idx2] = 2._prt * cluster_uz - uz_new; + + // set ids of merged particles so they will be removed + for (int j = 2; j < particles_in_bin; ++j){ + idcpu[indices[sorted_indices_data[i - j]]] = amrex::ParticleIdCpus::Invalid; + } + } + + // restart the tallies + particles_in_bin = 0; + total_weight = 0._prt; + total_energy = 0._prt; +#if !defined(WARPX_DIM_1D_Z) + cluster_x = 0_prt; +#endif +#if defined(WARPX_DIM_3D) + cluster_y = 0_prt; +#endif + cluster_z = 0_prt; + cluster_ux = 0_prt; + cluster_uy = 0_prt; + cluster_uz = 0_prt; + } + } + } + ); +} From 486e7939f0dc59b6a81278523093d05519973630 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 08:23:26 +0200 Subject: [PATCH 084/121] Remove friend class PML from WarpX class (#4874) --- Source/BoundaryConditions/PML.H | 4 +-- Source/BoundaryConditions/PML.cpp | 7 +++-- Source/Initialization/WarpXInitData.cpp | 42 +++++++++++++------------ Source/WarpX.H | 4 +-- Source/WarpX.cpp | 2 -- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 835877f2de9..42a96f3628a 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -136,8 +136,8 @@ private: class PML { public: - PML (int lev, - const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + PML (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, bool do_similar_dm_pml, const amrex::Geometry* geom, const amrex::Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 0b2806b3151..09088a24ada 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -542,7 +542,8 @@ MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) } } -PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& grid_dm, +PML::PML (const int lev, const BoxArray& grid_ba, + const DistributionMapping& grid_dm, const bool do_similar_dm_pml, const Geometry* geom, const Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, @@ -658,7 +659,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } DistributionMapping dm; - if (WarpX::do_similar_dm_pml) { + if (do_similar_dm_pml) { auto ng_sim = amrex::elemwiseMax(amrex::elemwiseMax(nge, ngb), ngf); dm = amrex::MakeSimilarDM(ba, grid_ba, grid_dm, ng_sim); } else { @@ -806,7 +807,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const BoxArray& cba = MakeBoxArray(is_single_box_domain, cdomain, *cgeom, grid_cba_reduced, cncells, do_pml_in_domain, do_pml_Lo, do_pml_Hi); DistributionMapping cdm; - if (WarpX::do_similar_dm_pml) { + if (do_similar_dm_pml) { auto ng_sim = amrex::elemwiseMax(amrex::elemwiseMax(nge, ngb), ngf); cdm = amrex::MakeSimilarDM(cba, grid_cba_reduced, grid_dm, ng_sim); } else { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5134a3aaa10..9dff4023f6d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -587,16 +587,17 @@ WarpX::InitPML () // Note: fill_guards_fields and fill_guards_current are both set to // zero (amrex::IntVect(0)) (what we do with damping BCs does not apply // to the PML, for example in the presence of mesh refinement patches) - pml[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), nullptr, - pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), - dt[0], nox_fft, noy_fft, noz_fft, grid_type, - do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, - do_pml_dive_cleaning, do_pml_divb_cleaning, - amrex::IntVect(0), amrex::IntVect(0), - guard_cells.ng_FieldSolver.max(), - v_particle_pml, - do_pml_Lo[0], do_pml_Hi[0]); + pml[0] = std::make_unique( + 0, boxArray(0), DistributionMap(0), do_similar_dm_pml, &Geom(0), nullptr, + pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), + dt[0], nox_fft, noy_fft, noz_fft, grid_type, + do_moving_window, pml_has_particles, do_pml_in_domain, + psatd_solution_type, J_in_time, rho_in_time, + do_pml_dive_cleaning, do_pml_divb_cleaning, + amrex::IntVect(0), amrex::IntVect(0), + guard_cells.ng_FieldSolver.max(), + v_particle_pml, + do_pml_Lo[0], do_pml_Hi[0]); #endif for (int lev = 1; lev <= finest_level; ++lev) @@ -625,16 +626,17 @@ WarpX::InitPML () // Note: fill_guards_fields and fill_guards_current are both set to // zero (amrex::IntVect(0)) (what we do with damping BCs does not apply // to the PML, for example in the presence of mesh refinement patches) - pml[lev] = std::make_unique(lev, boxArray(lev), DistributionMap(lev), - &Geom(lev), &Geom(lev-1), - pml_ncell, pml_delta, refRatio(lev-1), - dt[lev], nox_fft, noy_fft, noz_fft, grid_type, - do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, - amrex::IntVect(0), amrex::IntVect(0), - guard_cells.ng_FieldSolver.max(), - v_particle_pml, - do_pml_Lo[lev], do_pml_Hi[lev]); + pml[lev] = std::make_unique( + lev, boxArray(lev), DistributionMap(lev), do_similar_dm_pml, + &Geom(lev), &Geom(lev-1), + pml_ncell, pml_delta, refRatio(lev-1), + dt[lev], nox_fft, noy_fft, noz_fft, grid_type, + do_moving_window, pml_has_particles, do_pml_in_domain, + psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, + amrex::IntVect(0), amrex::IntVect(0), + guard_cells.ng_FieldSolver.max(), + v_particle_pml, + do_pml_Lo[lev], do_pml_Hi[lev]); } } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 9ca754ea1bb..7d2b8e45907 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -107,8 +107,6 @@ class WARPX_EXPORT WarpX { public: - friend class PML; - static WarpX& GetInstance (); static void ResetInstance (); @@ -1613,7 +1611,7 @@ private: int pml_has_particles = 0; int do_pml_j_damping = 0; int do_pml_in_domain = 0; - static int do_similar_dm_pml; + bool do_similar_dm_pml = true; bool do_pml_dive_cleaning; // default set in WarpX.cpp bool do_pml_divb_cleaning; // default set in WarpX.cpp amrex::Vector do_pml_Lo; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9a9903f013f..4b693d6cf99 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -212,8 +212,6 @@ int WarpX::n_current_deposition_buffer = -1; short WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; -int WarpX::do_similar_dm_pml = 1; - #ifdef AMREX_USE_GPU bool WarpX::do_device_synchronize = true; #else From 206e6d54a6969379103722be619eed3661ee33df Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 18:23:29 +0200 Subject: [PATCH 085/121] move field type struct in a separate header (#4875) --- Source/BoundaryConditions/PML.cpp | 2 + Source/BoundaryConditions/PML_RZ.cpp | 2 + .../WarpXFieldBoundaries.cpp | 4 +- Source/Diagnostics/BTDiagnostics.cpp | 2 + .../ComputeDiagFunctors/JFunctor.cpp | 3 ++ .../ComputeDiagFunctors/JdispFunctor.cpp | 2 + .../FlushFormats/FlushFormatCheckpoint.cpp | 2 + .../FlushFormats/FlushFormatPlotfile.cpp | 6 ++- Source/Diagnostics/FullDiagnostics.cpp | 3 ++ Source/Diagnostics/ParticleIO.cpp | 2 + .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 + .../ReducedDiags/ColliderRelevant.cpp | 2 + .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 2 + .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 2 + .../ReducedDiags/FieldMomentum.cpp | 2 + .../Diagnostics/ReducedDiags/FieldProbe.cpp | 2 + .../Diagnostics/ReducedDiags/FieldReduction.H | 20 +++++----- .../ReducedDiags/LoadBalanceCosts.cpp | 2 + .../ReducedDiags/ParticleExtrema.cpp | 2 + Source/Diagnostics/SliceDiagnostic.cpp | 5 ++- Source/FieldSolver/ElectrostaticSolver.cpp | 1 + Source/FieldSolver/Fields.H | 40 +++++++++++++++++++ .../HybridPICModel/HybridPICModel.cpp | 3 ++ .../MacroscopicProperties.cpp | 2 + Source/Particles/MultiParticleContainer.cpp | 3 ++ Source/WarpX.H | 38 +++--------------- Source/WarpX.cpp | 1 + 27 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 Source/FieldSolver/Fields.H diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 09088a24ada..25b34818dd1 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -10,6 +10,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" +#include "FieldSolver/Fields.H" #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif @@ -55,6 +56,7 @@ #endif using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 5eb30f43cca..faa7609b3ab 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -8,6 +8,7 @@ #include "PML_RZ.H" #include "BoundaryConditions/PML_RZ.H" +#include "FieldSolver/Fields.H" #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif @@ -33,6 +34,7 @@ #include using namespace amrex; +using namespace warpx::fields; PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, const amrex::Geometry* geom, const int ncell, const int do_pml_in_domain) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 0e50c560e7c..a5e0004325a 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -15,8 +15,10 @@ #include #include #include -using namespace amrex::literals; + using namespace amrex; +using namespace amrex::literals; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 40a76eac946..1cee9909226 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -14,6 +14,7 @@ #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" #include "ComputeDiagFunctors/BackTransformParticleFunctor.H" +#include "FieldSolver/Fields.H" #include "Utils/Algorithms/IsIn.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" @@ -46,6 +47,7 @@ #include using namespace amrex::literals; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index a8fd142815a..ebaec47b2f1 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -6,6 +6,7 @@ #include "JFunctor.H" +#include "FieldSolver/Fields.H" #include "Particles/MultiParticleContainer.H" #include "WarpX.H" @@ -14,6 +15,8 @@ #include #include +using namespace warpx::fields; + JFunctor::JFunctor (const int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index b73931814b6..aac5869da65 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -6,6 +6,7 @@ #include "JdispFunctor.H" #include "WarpX.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" @@ -15,6 +16,7 @@ #include using namespace amrex; +using namespace warpx::fields; JdispFunctor::JdispFunctor (int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 653ac0c26ba..27d0ad84b9c 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -5,6 +5,7 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FieldSolver/Fields.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" @@ -19,6 +20,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 7b10edf0d2b..94b9901b06e 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -1,16 +1,17 @@ #include "FlushFormatPlotfile.H" -#include "Particles/ParticleIO.H" +#include "FieldSolver/Fields.H" +#include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/Filter/FilterFunctors.H" #include "Particles/WarpXParticleContainer.H" +#include "Particles/ParticleIO.H" #include "Particles/PinnedMemoryParticleContainer.H" #include "Utils/Interpolate.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" -#include "Diagnostics/MultiDiagnostics.H" #include #include @@ -47,6 +48,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 63683690483..b25f899e29d 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -1,4 +1,5 @@ #include "FullDiagnostics.H" + #include "ComputeDiagFunctors/CellCenterFunctor.H" #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" @@ -10,6 +11,7 @@ #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FieldSolver/Fields.H" #include "FlushFormats/FlushFormat.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/IsIn.H" @@ -40,6 +42,7 @@ #include using namespace amrex::literals; +using namespace warpx::fields; FullDiagnostics::FullDiagnostics (int i, const std::string& name): Diagnostics{i, name}, diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 5cb18602ed9..c4164e383b1 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -7,6 +7,7 @@ * License: BSD-3-Clause-LBNL */ +#include "FieldSolver/Fields.H" #include "Particles/ParticleIO.H" #include "Particles/MultiParticleContainer.H" #include "Particles/PhysicalParticleContainer.H" @@ -41,6 +42,7 @@ #include using namespace amrex; +using namespace warpx::fields; void LaserParticleContainer::ReadHeader (std::istream& is) diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 7e66061fa48..2991831420e 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -8,6 +8,7 @@ #include "ChargeOnEB.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/Parser/ParserUtils.H" @@ -26,6 +27,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor ChargeOnEB::ChargeOnEB (const std::string& rd_name) diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 88dd0c0922b..434ab1d7949 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -8,6 +8,7 @@ #include "ColliderRelevant.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif @@ -58,6 +59,7 @@ #include using namespace amrex; +using namespace warpx::fields; ColliderRelevant::ColliderRelevant (const std::string& rd_name) : ReducedDiags{rd_name} diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 65b33341d1a..f4b4e2a39a1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -7,6 +7,7 @@ #include "FieldEnergy.H" +#include "FieldSolver/Fields.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -28,6 +29,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor FieldEnergy::FieldEnergy (const std::string& rd_name) diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 80979cd845e..24d9dda3ea6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -7,6 +7,7 @@ #include "FieldMaximum.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -38,6 +39,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor FieldMaximum::FieldMaximum (const std::string& rd_name) diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 901748bd6a9..f182acd5ba2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -7,6 +7,7 @@ #include "FieldMomentum.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" @@ -37,6 +38,7 @@ #include using namespace amrex; +using namespace warpx::fields; FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 4be5a928d96..7364d5989f1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -7,6 +7,7 @@ #include "FieldProbe.H" #include "FieldProbeParticleContainer.H" +#include "FieldSolver/Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/UpdatePosition.H" @@ -44,6 +45,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index a2813bf51e8..a15d6489e2c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -9,6 +9,7 @@ #define WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDREDUCTION_H_ #include "ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "WarpX.H" #include @@ -36,7 +37,6 @@ #include #include - /** * This class contains a function that computes an arbitrary reduction of the fields. The function * used in the reduction is defined by an input file parser expression and the reduction operation @@ -99,15 +99,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); - const amrex::MultiFab & jx = warpx.getField(FieldType::current_fp, lev,0); - const amrex::MultiFab & jy = warpx.getField(FieldType::current_fp, lev,1); - const amrex::MultiFab & jz = warpx.getField(FieldType::current_fp, lev,2); + const amrex::MultiFab & Ex = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & jx = warpx.getField(warpx::fields::FieldType::current_fp, lev,0); + const amrex::MultiFab & jy = warpx.getField(warpx::fields::FieldType::current_fp, lev,1); + const amrex::MultiFab & jz = warpx.getField(warpx::fields::FieldType::current_fp, lev,2); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index e5e8a7adc07..882c94b12eb 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -7,6 +7,7 @@ #include "LoadBalanceCosts.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -35,6 +36,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 79b66a92e66..0cc5429be7a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -11,6 +11,7 @@ #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif +#include "FieldSolver/Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" #include "Particles/MultiParticleContainer.H" @@ -51,6 +52,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor ParticleExtrema::ParticleExtrema (const std::string& rd_name) diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index aaadac52610..14a2d633a0c 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -7,8 +7,9 @@ */ #include "SliceDiagnostic.H" -#include "WarpX.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" +#include "WarpX.H" #include #include @@ -40,7 +41,7 @@ #include using namespace amrex; - +using namespace warpx::fields; /* \brief * The functions creates the slice for diagnostics based on the user-input. diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index d0818959b30..5bb50ada278 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -55,6 +55,7 @@ #include using namespace amrex; +using namespace warpx::fields; void WarpX::ComputeSpaceChargeField (bool const reset_fields) diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H new file mode 100644 index 00000000000..76ff9a74f7b --- /dev/null +++ b/Source/FieldSolver/Fields.H @@ -0,0 +1,40 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_FIELDS_H_ +#define WARPX_FIELDS_H_ + +namespace warpx::fields +{ + enum struct FieldType : int + { + Efield_aux, + Bfield_aux, + Efield_fp, + Bfield_fp, + current_fp, + current_fp_nodal, + rho_fp, + F_fp, + G_fp, + phi_fp, + vector_potential_fp, + Efield_cp, + Bfield_cp, + current_cp, + rho_cp, + F_cp, + G_cp, + edge_lengths, + face_areas, + Efield_avg_fp, + Bfield_avg_fp, + Efield_avg_cp, + Bfield_avg_cp + }; +} + +#endif //WARPX_FIELDS_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 6a688ce38b3..8979036fbea 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -9,7 +9,10 @@ #include "HybridPICModel.H" +#include "FieldSolver/Fields.H" + using namespace amrex; +using namespace warpx::fields; HybridPICModel::HybridPICModel ( int nlevs_max ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index c60a0b4c4d5..d9a8ec71fe1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -1,5 +1,6 @@ #include "MacroscopicProperties.H" +#include "FieldSolver/Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -26,6 +27,7 @@ #include using namespace amrex; +using namespace warpx::fields; MacroscopicProperties::MacroscopicProperties () { diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index dda32467925..e4823bfaf6f 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -10,6 +10,8 @@ * License: BSD-3-Clause-LBNL */ #include "MultiParticleContainer.H" + +#include "FieldSolver/Fields.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" @@ -80,6 +82,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/WarpX.H b/Source/WarpX.H index 7d2b8e45907..d19843ca636 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -38,6 +38,7 @@ #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" @@ -75,33 +76,6 @@ #include #include -enum struct FieldType : int -{ - Efield_aux, - Bfield_aux, - Efield_fp, - Bfield_fp, - current_fp, - current_fp_nodal, - rho_fp, - F_fp, - G_fp, - phi_fp, - vector_potential_fp, - Efield_cp, - Bfield_cp, - current_cp, - rho_cp, - F_cp, - G_cp, - edge_lengths, - face_areas, - Efield_avg_fp, - Bfield_avg_fp, - Efield_avg_cp, - Bfield_avg_cp -}; - class WARPX_EXPORT WarpX : public amrex::AmrCore { @@ -492,7 +466,7 @@ public: * \return true if the field is initialized, false otherwise */ [[nodiscard]] bool - isFieldInitialized (FieldType field_type, int lev, int direction = 0) const; + isFieldInitialized (warpx::fields::FieldType field_type, int lev, int direction = 0) const; /** * \brief @@ -505,7 +479,7 @@ public: * \return the pointer to an amrex::MultiFab containing the field data */ [[nodiscard]] amrex::MultiFab* - getFieldPointer (FieldType field_type, int lev, int direction = 0) const; + getFieldPointer (warpx::fields::FieldType field_type, int lev, int direction = 0) const; /** * \brief @@ -517,7 +491,7 @@ public: * \return an array of three pointers amrex::MultiFab* containing the field data */ [[nodiscard]] std::array - getFieldPointerArray (FieldType field_type, int lev) const; + getFieldPointerArray (warpx::fields::FieldType field_type, int lev) const; /** * \brief @@ -530,7 +504,7 @@ public: * \return a constant refernce to an amrex::MultiFab containing the field data */ [[nodiscard]] const amrex::MultiFab& - getField(FieldType field_type, int lev, int direction = 0) const; + getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -1601,7 +1575,7 @@ private: * \return the pointer to an amrex::MultiFab containing the field data */ [[nodiscard]] amrex::MultiFab* - getFieldPointerUnchecked (FieldType field_type, int lev, int direction = 0) const; + getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; // PML int do_pml = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4b693d6cf99..9a29d7c2354 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -85,6 +85,7 @@ #include using namespace amrex; +using namespace warpx::fields; int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; From 28434e957bf8104479a4774b5653e46e391b572f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 18:45:15 +0200 Subject: [PATCH 086/121] remove call to WarpX::GetInstance() from FilterCreateTransformFromFAB.H (#4873) --- Source/Particles/MultiParticleContainer.cpp | 4 +- .../FilterCreateTransformFromFAB.H | 38 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index e4823bfaf6f..fc496217388 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -1399,10 +1399,12 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; + const amrex::Geometry& geom_level_zero = warpx.Geom(level_0); + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, - Transform); + Transform, geom_level_zero); setNewParticleIDs(dst_ele_tile, np_ele_dst, num_added); setNewParticleIDs(dst_pos_tile, np_pos_dst, num_added); diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 1fe44bdfa7f..8a83c60b221 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -9,7 +9,6 @@ #define WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" -#include "WarpX.H" #include #include @@ -40,6 +39,7 @@ * \param[in] create1 callable that defines what will be done for the create step for dst1. * \param[in] create2 callable that defines what will be done for the create step for dst2. * \param[in] transform callable that defines the transformation to apply on dst1 and dst2. + * \param[in] geom_lev_zero the geometry object associated to level zero * * \return num_added the number of particles that were written to dst1 and dst2. */ @@ -51,34 +51,30 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, const FAB *src_FAB, const Index* mask, const Index dst1_index, const Index dst2_index, CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept + TransFunc&& transform, const amrex::Geometry& geom_lev_zero) noexcept { using namespace amrex; const auto ncells = box.volume(); if (ncells == 0) { return 0; } - auto & warpx = WarpX::GetInstance(); - const int level_0 = 0; - Geometry const & geom = warpx.Geom(level_0); - constexpr int spacedim = AMREX_SPACEDIM; #if defined(WARPX_DIM_1D_Z) - const Real zlo_global = geom.ProbLo(0); - const Real dz = geom.CellSize(0); + const Real zlo_global = geom_lev_zero.ProbLo(0); + const Real dz = geom_lev_zero.CellSize(0); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const Real xlo_global = geom.ProbLo(0); - const Real dx = geom.CellSize(0); - const Real zlo_global = geom.ProbLo(1); - const Real dz = geom.CellSize(1); + const Real xlo_global = geom_lev_zero.ProbLo(0); + const Real dx = geom_lev_zero.CellSize(0); + const Real zlo_global = geom_lev_zero.ProbLo(1); + const Real dz = geom_lev_zero.CellSize(1); #elif defined(WARPX_DIM_3D) - const Real xlo_global = geom.ProbLo(0); - const Real dx = geom.CellSize(0); - const Real ylo_global = geom.ProbLo(1); - const Real dy = geom.CellSize(1); - const Real zlo_global = geom.ProbLo(2); - const Real dz = geom.CellSize(2); + const Real xlo_global = geom_lev_zero.ProbLo(0); + const Real dx = geom_lev_zero.CellSize(0); + const Real ylo_global = geom_lev_zero.ProbLo(1); + const Real dy = geom_lev_zero.CellSize(1); + const Real zlo_global = geom_lev_zero.ProbLo(2); + const Real dz = geom_lev_zero.CellSize(2); #endif const auto arrNumPartCreation = src_FAB->array(); @@ -199,6 +195,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, * \param[in] create1 callable that defines what will be done for the create step for dst1. * \param[in] create2 callable that defines what will be done for the create step for dst2. * \param[in] transform callable that defines the transformation to apply on dst1 and dst2. + * \param[in] geom_lev_zero the geometry object associated to level zero * * \return num_added the number of particles that were written to dst1 and dst2. */ @@ -209,7 +206,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc && transform) noexcept + TransFunc && transform, const amrex::Geometry& geom_lev_zero) noexcept { using namespace amrex; @@ -241,7 +238,8 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), - std::forward(transform)); + std::forward(transform), + geom_lev_zero); } #endif // WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ From 7e651179b2b30e01b2011c7d5de3aef86e21ee3a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 20:25:39 +0200 Subject: [PATCH 087/121] remove calls to WarpX:GetInstance() from MacroscopicProperties.cpp (#4872) --- .../MacroscopicProperties.H | 25 +++++++++++-- .../MacroscopicProperties.cpp | 35 ++++++++----------- Source/Initialization/WarpXInitData.cpp | 11 +++++- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 205b009b2eb..129b2e2f782 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -13,8 +13,11 @@ #include "Utils/WarpXConst.H" #include +#include +#include #include #include +#include #include #include #include @@ -33,8 +36,26 @@ public: MacroscopicProperties (); // constructor /** Read user-defined macroscopic properties. Called in constructor. */ void ReadParameters (); - /** Initialize multifabs storing macroscopic multifabs */ - void InitData (); + + /** + * \brief Initialize multifabs storing macroscopic multifabs + * + * @param[in] ba the box array associated to the multifabs E and B + * @param[in] dmap the distribution mapping + * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B + * @param[in] geom the geometry + * @param[in] Ex_stag staggering of the Ex field + * @param[in] Ey_stag staggering of the Ey field + * @param[in] Ez_stag staggering of the Ez field + */ + void InitData ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dmap, + const amrex::IntVect& ng_EB_alloc, + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag); /** return MultiFab, sigma (conductivity) of the medium. */ amrex::MultiFab& getsigma_mf () {return (*m_sigma_mf);} diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index d9a8ec71fe1..1467c7a8c0c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -3,18 +3,14 @@ #include "FieldSolver/Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" -#include "WarpX.H" #include #include -#include #include -#include #include #include #include -#include #include #include #include @@ -123,16 +119,17 @@ MacroscopicProperties::ReadParameters () } void -MacroscopicProperties::InitData () +MacroscopicProperties::InitData ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dmap, + const amrex::IntVect& ng_EB_alloc, + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag) { amrex::Print() << Utils::TextMsg::Info("we are in init data of macro"); - auto & warpx = WarpX::GetInstance(); - // Get BoxArray and DistributionMap of warpx instance. - const int lev = 0; - const amrex::BoxArray ba = warpx.boxArray(lev); - const amrex::DistributionMapping dmap = warpx.DistributionMap(lev); - const amrex::IntVect ng_EB_alloc = warpx.getngEB(); // Define material property multifabs using ba and dmap from WarpX instance // sigma is cell-centered MultiFab m_sigma_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); @@ -148,7 +145,7 @@ MacroscopicProperties::InitData () } else if (m_sigma_s == "parse_sigma_function") { InitializeMacroMultiFabUsingParser(m_sigma_mf.get(), m_sigma_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } // Initialize epsilon if (m_epsilon_s == "constant") { @@ -158,7 +155,7 @@ MacroscopicProperties::InitData () } else if (m_epsilon_s == "parse_epsilon_function") { InitializeMacroMultiFabUsingParser(m_eps_mf.get(), m_epsilon_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } // In the Maxwell solver, `epsilon` is used in the denominator. @@ -175,16 +172,14 @@ MacroscopicProperties::InitData () } else if (m_mu_s == "parse_mu_function") { InitializeMacroMultiFabUsingParser(m_mu_mf.get(), m_mu_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } - amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); - amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); - amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); + const amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); + const amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); + const amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { sigma_IndexType[idim] = sigma_stag[idim]; diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 9dff4023f6d..47eeaa41f86 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -474,7 +474,16 @@ WarpX::InitData () BuildBufferMasks(); if (WarpX::em_solver_medium==1) { - m_macroscopic_properties->InitData(); + const int lev_zero = 0; + m_macroscopic_properties->InitData( + boxArray(lev_zero), + DistributionMap(lev_zero), + getngEB(), + Geom(lev_zero), + getField(FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + ); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { From 10f6f675a2e150fdabbbd6339f049ad61b975e82 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Apr 2024 13:07:33 -0700 Subject: [PATCH 088/121] TNSA Example: Link Domain Decomposition (#4885) - link domain decomposition for `numprocs` option - remove outdated warning --- .../usage/workflows/domain_decomposition.rst | 6 ++++++ .../Physics_applications/laser_ion/README.rst | 21 +++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Docs/source/usage/workflows/domain_decomposition.rst b/Docs/source/usage/workflows/domain_decomposition.rst index 1340ecc10d9..201f2dcbbc5 100644 --- a/Docs/source/usage/workflows/domain_decomposition.rst +++ b/Docs/source/usage/workflows/domain_decomposition.rst @@ -5,6 +5,9 @@ Domain Decomposition WarpX relies on a spatial domain decomposition for MPI parallelization. It provides two different ways for users to specify this decomposition, a `simple` way recommended for most users, and a `flexible` way recommended if more control is desired. The `flexible` method is required for dynamic load balancing to be useful. + +.. _usage_domain_decomposition-simple: + 1. Simple Method ---------------- @@ -14,6 +17,9 @@ The first and simplest method is to provide the ``warpx.numprocs = nx ny nz`` pa If ``warpx.numprocs`` is *not* specified, WarpX will fall back on using the ``amr.max_grid_size`` and ``amr.blocking_factor`` parameters, described below. + +.. _usage_domain_decomposition-general: + 2. More General Method ---------------------- diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index 18a427aa1c7..29862a30518 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -15,12 +15,6 @@ Although laser-ion acceleration requires full 3D modeling for adequate descripti This includes spatial and temporal resolution, but also the number of macro-particles per cell representing the target density for proper phase space sampling. You will need a computing cluster for adequate resolution of the target density, see comments in the input file. -.. warning:: - - It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. - The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. - Not setting these parameters can quickly lead to memory overflows. - Run --- @@ -30,7 +24,12 @@ This example can be run **either** as: * **Python** script: ``mpiexec -n 2 python3 PICMI_inputs_2d.py`` or * WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_2d`` -For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. +.. tip:: + + For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. + + The input option ``warpx_numprocs`` / ``warpx.numprocs`` needs to be adjusted for parallel :ref:`domain decomposition `, to match the number of MPI ranks used. + In order to use dynamic load balancing, use the :ref:`more general method ` of setting blocks. .. tab-set:: @@ -53,18 +52,18 @@ Analyze .. _fig-tnsa-ps-electrons-pinhole: .. figure:: https://user-images.githubusercontent.com/5416860/295003882-c755fd47-4bb3-4439-9319-c48214cbaafd.png - :alt: Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + :alt: Longitudinal phase space of forward-moving electrons in a 2 degree opening angle. :width: 100% - Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + Longitudinal phase space of forward-moving electrons in a 2 degree opening angle. .. _fig-tnsa-ps-protons-pinhole: .. figure:: https://user-images.githubusercontent.com/5416860/295003988-dea3dfb7-0d55-4616-b32d-061fb429f9ac.png - :alt: Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + :alt: Longitudinal phase space of forward-moving protons in a 2 degree opening angle. :width: 100% - Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + Longitudinal phase space of forward-moving protons in a 2 degree opening angle. Time-resolved phase electron space analysis as in :numref:`fig-tnsa-ps-electrons-pinhole` gives information about, e.g., how laser energy is locally converted into electron kinetic energy. Later in time, ion phase spaces like :numref:`fig-tnsa-ps-protons-pinhole` can reveal where accelerated ion populations originate. From 94871c7162b835e2cfff699abbd80d85f3156fac Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:04:24 -0700 Subject: [PATCH 089/121] fix failing `development` builds (#4890) --- Source/Initialization/WarpXInitData.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 47eeaa41f86..e0c095918fa 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -480,9 +480,9 @@ WarpX::InitData () DistributionMap(lev_zero), getngEB(), Geom(lev_zero), - getField(FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), - getField(FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), - getField(FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() ); } From ed7e824911da5b25c9bc185dc3c164322156ee5e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Apr 2024 23:48:31 -0700 Subject: [PATCH 090/121] Great Lakes Cluster (UMich) (#4869) * Great Lakes Cluster (UMich) Document how to use the Great Lakes Cluster at the University of Michigan. * Fix c-blosc2 typos * Parallel HDF5 for CUDA-aware MPI New module added on the system. * Fix GPU Visibility Ensure one MPI rank sees exactly one, unique GPU. * Add `#SBATCH --gpu-bind=single:1` * Add clean-up message. --- Docs/source/install/hpc.rst | 1 + Docs/source/install/hpc/greatlakes.rst | 240 ++++++++++++++++++ .../greatlakes-umich/greatlakes_v100.sbatch | 38 +++ .../greatlakes_v100_warpx.profile.example | 62 +++++ .../install_v100_dependencies.sh | 133 ++++++++++ 5 files changed, 474 insertions(+) create mode 100644 Docs/source/install/hpc/greatlakes.rst create mode 100644 Tools/machines/greatlakes-umich/greatlakes_v100.sbatch create mode 100644 Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example create mode 100755 Tools/machines/greatlakes-umich/install_v100_dependencies.sh diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index a7b0f636b56..5efdeae39a8 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -51,6 +51,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/spock hpc/summit hpc/taurus + hpc/greatlakes .. tip:: diff --git a/Docs/source/install/hpc/greatlakes.rst b/Docs/source/install/hpc/greatlakes.rst new file mode 100644 index 00000000000..d238d498e5c --- /dev/null +++ b/Docs/source/install/hpc/greatlakes.rst @@ -0,0 +1,240 @@ +.. _building-greatlakes: + +Great Lakes (UMich) +=================== + +The `Great Lakes cluster `_ is located at University of Michigan. +The cluster has various partitions, including `GPU nodes and CPU nodes `__. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Great Lakes user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ (`documentation `__) +* `Filesystems `__: + + * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (80GB) + * ``/scratch``: per-project `production directory `__; very fast for parallel jobs; purged every 60 days (10TB default) + + +.. _building-greatlakes-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Great Lakes, you can run either on GPU nodes with `fast V100 GPUs (recommended), the even faster A100 GPUs (only a few available) or CPU nodes `__. + +.. tab-set:: + + .. tab-item:: V100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/greatlakes_v100_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example $HOME/greatlakes_v100_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``iloveplasma``, then run ``nano $HOME/greatlakes_v100_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="iloveplasma" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to Great Lakes, activate these environment settings: + + .. code-block:: bash + + source $HOME/greatlakes_v100_warpx.profile + + Finally, since Great Lakes does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/greatlakes-umich/install_v100_dependencies.sh + source ${HOME}/sw/greatlakes/v100/venvs/warpx-v100/bin/activate + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/install_v100_dependencies.sh + :language: bash + + + .. tab-item:: A100 Nodes + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + .. note:: + + This section is TODO. + + +.. _building-greatlakes-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + + .. tab-item:: V100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100 + + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100 -j 8 + + The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. + Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100_py + + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100_py -j 8 --target pip_install + + + .. tab-item:: A100 Nodes + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + .. note:: + + This section is TODO. + +Now, you can :ref:`submit Great Lakes compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit greatlakes jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch``. + + +.. _building-greatlakes-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the greatlakes_v100_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. + + +.. _running-cpp-greatlakes: + +Running +------- + +.. tab-set:: + + .. tab-item:: V100 (16GB) GPUs + + The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly) on the supercomputer Great Lakes at University of Michigan. + This partition has `20 nodes, each with two V100 GPUs `__. + + Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. + Note that we run one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/greatlakes_v100.sbatch + :language: bash + :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch``. + + To run a simulation, copy the lines above to a file ``greatlakes_v100.sbatch`` and run + + .. code-block:: bash + + sbatch greatlakes_v100.sbatch + + to submit the job. + + + .. tab-item:: A100 (80GB) GPUs + + This partition has `2 nodes, each with four A100 GPUs `__ that provide 80 GB HBM per A100 GPU. + To the user, each node will appear as if it has 8 A100 GPUs with 40 GB memory each. + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + The Great Lakes CPU partition as up to `455 nodes `__, each with 2x Intel Xeon Gold 6154 CPUs and 180 GB RAM. + + .. note:: + + This section is TODO. + + +.. _post-processing-greatlakes: + +Post-Processing +--------------- + +For post-processing, many users prefer to use the online `Jupyter service `__ (`documentation `__) that is directly connected to the cluster's fast filesystem. + +.. note:: + + This section is a stub and contributions are welcome. + We can document further details, e.g., which recommended post-processing Python software to install or how to customize Jupyter kernels here. diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch new file mode 100644 index 00000000000..0353c08456f --- /dev/null +++ b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch @@ -0,0 +1,38 @@ +#!/bin/bash -l + +# Copyright 2024 The WarpX Community +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +#SBATCH -t 00:10:00 +#SBATCH -N 1 +#SBATCH -J WarpX +#SBATCH -A +#SBATCH --partition=gpu +#SBATCH --exclusive +#SBATCH --ntasks-per-node=2 +#SBATCH --cpus-per-task=20 +#SBATCH --gpus-per-task=v100:1 +#SBATCH --gpu-bind=single:1 +#SBATCH -o WarpX.o%j +#SBATCH -e WarpX.e%j + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=inputs + +# threads for OpenMP and threaded compressors per MPI rank +# per node are 2x 2.4 GHz Intel Xeon Gold 6148 +# note: the system seems to only expose cores (20 per socket), +# not hyperthreads (40 per socket) +export SRUN_CPUS_PER_TASK=20 +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} + +# GPU-aware MPI optimizations +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# run WarpX +srun --cpu-bind=cores \ + ${EXE} ${INPUTS} ${GPU_AWARE_MPI} \ + > output.txt diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example new file mode 100644 index 00000000000..98476eecfab --- /dev/null +++ b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example @@ -0,0 +1,62 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi + +# required dependencies +module purge +module load gcc/10.3.0 +module load cuda/12.1.1 +module load cmake/3.26.3 +module load openblas/0.3.23 +module load openmpi/4.1.6-cuda + +# optional: for QED support +module load boost/1.78.0 + +# optional: for openPMD and PSATD+RZ support +module load phdf5/1.12.1 + +SW_DIR="${HOME}/sw/greatlakes/v100" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc2-2.14.4:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.0:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc2-2.14.4/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.0/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH + +export PATH=${SW_DIR}/adios2-2.10.0/bin:${PATH} + +# optional: for Python bindings or libEnsemble +module load python/3.12.1 + +if [ -d "${SW_DIR}/venvs/warpx-v100" ] +then + source ${SW_DIR}/venvs/warpx-v100/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --partition=gpu --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --partition=gpu --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" + +# optimize CUDA compilation for V100 +export AMREX_CUDA_ARCH=7.0 + +# optimize CPU microarchitecture for Intel Xeon Gold 6148 +export CXXFLAGS="-march=skylake-avx512" +export CFLAGS="-march=skylake-avx512" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh new file mode 100755 index 00000000000..7235b18c99f --- /dev/null +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was greatlakes_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your greatlakes_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +echo "Cleaning up prior installation directory... This may take several minutes." +SW_DIR="${HOME}/sw/greatlakes/v100" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d $HOME/src/c-blosc2 ] +then + cd $HOME/src/c-blosc2 + git fetch --prune + git checkout v2.14.4 + cd - +else + git clone -b v2.14.4 https://github.com/Blosc/c-blosc2.git $HOME/src/c-blosc2 +fi +rm -rf $HOME/src/c-blosc2-v100-build +cmake -S $HOME/src/c-blosc2 -B ${build_dir}/c-blosc2-v100-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_FUZZERS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc2-2.14.4 +cmake --build ${build_dir}/c-blosc2-v100-build --target install --parallel 8 +rm -rf ${build_dir}/c-blosc2-v100-build + +# ADIOS2 +if [ -d $HOME/src/adios2 ] +then + cd $HOME/src/adios2 + git fetch --prune + git checkout v2.10.0 + cd - +else + git clone -b v2.10.0 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 +fi +rm -rf $HOME/src/adios2-v100-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-v100-build -DADIOS2_USE_Blosc2=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 +cmake --build ${build_dir}/adios2-v100-build --target install -j 8 +rm -rf ${build_dir}/adios2-v100-build + +# BLAS++ (for PSATD+RZ) +if [ -d $HOME/src/blaspp ] +then + cd $HOME/src/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp +fi +rm -rf $HOME/src/blaspp-v100-build +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-v100-build --target install --parallel 8 +rm -rf ${build_dir}/blaspp-v100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d $HOME/src/lapackpp ] +then + cd $HOME/src/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp +fi +rm -rf $HOME/src/lapackpp-v100-build +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-v100-build --target install --parallel 8 +rm -rf ${build_dir}/lapackpp-v100-build + + +# Python ###################################################################### +# +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade virtualenv +python3 -m pip cache purge +rm -rf ${SW_DIR}/venvs/warpx-v100 +python3 -m venv ${SW_DIR}/venvs/warpx-v100 +source ${SW_DIR}/venvs/warpx-v100/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies +python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch # CUDA 12 compatible wheel +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} From 07f239c70fe3678d5b438012df0853222b42cc80 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Apr 2024 23:51:22 -0700 Subject: [PATCH 091/121] Open Governance Model (#4743) * [Draft] Open Governance Model This is a first draft of the open governance model for WarpX, developed after discussion with the current leadership team. The goal of the governance model is to: - document and make transparent our decision structure, - structure and openly credit sensible roles, - clarify appointment and offboarding, - encourage participation, engagement and growth. Thoughts that guided the initial document include: - accurately reflecting our successful project workflows, - evolving it for shared responsibiliy and co-ownership, - avoiding unnecessary overheads and deadlocks, - be flexible enough to enable later introduction of sub-roles and further formalization, depending on community growth. * Review Session today * Clarify SC permissions are for each member. * Fix Typo: Groenewald Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Added TC Veto Overwrite Rule * Documentation & Link to (Access) Developer Meeting Notes * Added Jean-Luc also to TC --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Docs/source/governance.rst | 1 + Docs/source/index.rst | 1 + GOVERNANCE.rst | 144 +++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 120000 Docs/source/governance.rst create mode 100644 GOVERNANCE.rst diff --git a/Docs/source/governance.rst b/Docs/source/governance.rst new file mode 120000 index 00000000000..1b1b99778ab --- /dev/null +++ b/Docs/source/governance.rst @@ -0,0 +1 @@ +../../GOVERNANCE.rst \ No newline at end of file diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 9f2d2ff038e..89bc2e417cb 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -150,4 +150,5 @@ Epilogue :hidden: glossary + governance acknowledgements diff --git a/GOVERNANCE.rst b/GOVERNANCE.rst new file mode 100644 index 00000000000..b5253b80f9f --- /dev/null +++ b/GOVERNANCE.rst @@ -0,0 +1,144 @@ +.. _governance: + +WarpX Governance +================ + +WarpX is led in an open governance model, described in this file. + + +Steering Committee +------------------ + +Current Roster +^^^^^^^^^^^^^^ + +- Jean-Luc Vay (chair) +- Remi Lehe +- Axel Huebl + +See: `GitHub team `__ + +Role +^^^^ + +Members of the steering committee (SC) can change organizational settings, do administrative operations such as rename/move/archive repositories, change branch protection rules, etc. +SC members can call votes for decisions (technical or governance). + +The SC can veto decisions of the technical committee (TC) by voting in the SC. +The TC can overwrite a veto with a 2/3rd majority vote in the TC. +Decisions are documented in the `weekly developer meeting notes `__ and/or on the GitHub repository. + +The SC can change the governance structure, but only in a unanimous vote. + +Decision Process +^^^^^^^^^^^^^^^^ + +Decision of the SC usually happen in the weekly developer meetings, via e-mail or public chat. + +Decisions are made in a non-confidential manner, by majority on the cast votes of SC members. +Votes can be cast in asynchronous manner, e.g., over the time of 1-2 weeks. +In tie situations, the chair of the SC acts as the tie breaker. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +Appointed by current SC members in an unanimous vote. +As a SC member, regularly attending and contributing to the weekly developer meetings is expected. + +SC members can resign or be removed by majority vote, e.g., due to inactivity, bad acting or other reasons. + + +Technical Committee +------------------- + +Current Roster +^^^^^^^^^^^^^^ + +- Luca Fedeli +- Roelof Groenewald +- David Grote +- Axel Huebl +- Revathi Jambunathan +- Remi Lehe +- Andrew Myers +- Maxence Thévenet +- Jean-Luc Vay +- Weiqun Zhang +- Edoardo Zoni + +See: `GitHub team `__ + +Role +^^^^ + +The technical committee (TC) is the core governance body, where under normal operations most ideas are discussed and decisions are made. +Individual TC members can approve and merge code changes. +Usually, they seek approval by another maintainer for their own changes, too. +TC members lead - and weigh in on - technical discussions and, if needed, can call for a vote between TC members for a technical decision. +TC members merge/close PRs and issues, and moderate (including block/mute) bad actors. +The TC can propose governance changes to the SC. + + +Decision Process +^^^^^^^^^^^^^^^^ + +Discussion in the TC usually happens in the weekly developer meetings. + +If someone calls for a vote to make a decision: majority based on the cast votes; we need 50% of the committee participating to vote. In the absence of a quorum, the SC will decide according to its voting rules. + +Votes are cast in a non-confidential manner. +Decisions are documented in the `weekly developer meeting notes `__ and/or on the GitHub repository. + +TC members can individually appoint new contributors, unless a vote is called on an individual. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +TC members are the maintainers of WarpX. +As a TC member, regularly attending and contributing to the weekly developer meetings is expected. + +One is appointed to the TC by the steering committee, in a unanimous vote, or by majority vote of the TC. The SC can veto appointments. +Steering committee members can also be TC members. + +TC members can resign or be removed by majority vote by either TC or SC, e.g., due to inactivity, bad acting or other reasons. + + +Contributors +------------ + +Current Roster +^^^^^^^^^^^^^^ + +See: `GitHub team `__ + +Role +^^^^ + +Contributors are valuable, vetted developers of WarpX. +Contributions can be in many forms and not all need to be code contributions. +Examples include code pull requests, support in issues & user discussions, writing and updating documentation, writing tutorials, visualizations, R&D on algorithms, testing and benchmarking, etc. +Contributors can participate in developer meetings and weigh in on discussions. +Contributors can "triage" (add labels) to pull requests, issues, and GitHub discussion pages. +Contributors can comment and review PRs (but not merge). + +Decision Process +^^^^^^^^^^^^^^^^ + +Contributors can individually decide on classification (triage) of pull requests, issues, and GitHub discussion pages. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +Appointed after contributing to WarpX (see above) by any member of the TC. + +The role can be lost by resigning or by decision of an individual TC or SC member, e.g., due to inactivity, bad acting or other. + + +Former Members +-------------- + +"Former members" are the giants on whose shoulders we stand. +But, for the purpose of WarpX governance, they are *not* tracked as a governance role in WarpX. +Instead, former (e.g., inactive) contributors are acknowledged separately in GitHub contributor tracking, the WarpX documentation, references, citable Zenodo archives of releases, etc. as appropriate. + +Former members of SC, TC and Contributors are not kept in the roster, since committee role rosters shall reflect currently active members and the responsible governance body. From bc726a5f4d014cbf3551a14061db48d15be3684d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 25 Apr 2024 09:02:41 -0700 Subject: [PATCH 092/121] openPMD-api: 0.15.2 (#4847) * openPMD-api: 0.15.2 This patch release mostly improves reads. https://openpmd-api.readthedocs.io/en/0.15.2/install/changelog.html#id1 * 0.15.1+ - technically also works - we forgot to bump the installed version in 0.15.2 --- .github/workflows/dependencies/icc.sh | 2 +- Docs/source/install/cmake.rst | 2 +- Docs/source/usage/parameters.rst | 14 +++++++------- cmake/dependencies/openPMD.cmake | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index 74f0db4b46a..fae6e22d45a 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -58,7 +58,7 @@ export CEI_TMP="/tmp/cei" CXX=$(which icpc) CC=$(which icc) \ cmake-easyinstall \ --prefix=/usr/local \ - git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ + git+https://github.com/openPMD/openPMD-api.git@0.15.2 \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 2e66c95a4d6..006cd843b31 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -125,7 +125,7 @@ CMake Option Default & Values Des ``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` ``WarpX_openpmd_src`` *None* Path to openPMD-api source directory (preferred if set) ``WarpX_openpmd_repo`` ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from -``WarpX_openpmd_branch`` ``0.15.1`` Repository branch for ``WarpX_openpmd_repo`` +``WarpX_openpmd_branch`` ``0.15.2`` Repository branch for ``WarpX_openpmd_repo`` ``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD-api library if set to ``OFF`` ``WarpX_picsar_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 9f88f2d8c6c..5f97730e880 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2558,18 +2558,18 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. * ``.openpmd_encoding`` (optional, ``v`` (variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. - openPMD `file output encoding `__. + openPMD `file output encoding `__. File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). - ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. + ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. Default: ``f`` (full diagnostics) * ``.adios2_operator.type`` (``zfp``, ``blosc``) optional, - `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. * ``.adios2_operator.parameters.*`` optional, - `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. - A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): + A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): .. code-block:: text @@ -2588,11 +2588,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a .adios2_operator.parameters.precision = 3 * ``.adios2_engine.type`` (``bp4``, ``sst``, ``ssc``, ``dataman``) optional, - `ADIOS2 Engine type `__ for `openPMD `_ data dumps. + `ADIOS2 Engine type `__ for `openPMD `_ data dumps. See full list of engines at `ADIOS2 readthedocs `__ * ``.adios2_engine.parameters.*`` optional, - `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. + `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. An example for parameters for the BP engine are setting the number of writers (``NumAggregators``), transparently redirecting data to burst buffers etc. A detailed list of engine-specific parameters are available at the official `ADIOS2 documentation `__ diff --git a/cmake/dependencies/openPMD.cmake b/cmake/dependencies/openPMD.cmake index 93599d7505e..f58d37ee92e 100644 --- a/cmake/dependencies/openPMD.cmake +++ b/cmake/dependencies/openPMD.cmake @@ -13,7 +13,7 @@ function(find_openpmd) if(WarpX_openpmd_internal OR WarpX_openpmd_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - # see https://openpmd-api.readthedocs.io/en/0.15.1/dev/buildoptions.html + # see https://openpmd-api.readthedocs.io/en/0.15.2/dev/buildoptions.html set(openPMD_USE_ADIOS1 OFF CACHE INTERNAL "") set(openPMD_USE_MPI ${WarpX_MPI} CACHE INTERNAL "") set(openPMD_USE_PYTHON OFF CACHE INTERNAL "") @@ -92,7 +92,7 @@ if(WarpX_OPENPMD) set(WarpX_openpmd_repo "https://github.com/openPMD/openPMD-api.git" CACHE STRING "Repository URI to pull and build openPMD-api from if(WarpX_openpmd_internal)") - set(WarpX_openpmd_branch "0.15.1" + set(WarpX_openpmd_branch "0.15.2" CACHE STRING "Repository branch for WarpX_openpmd_repo if(WarpX_openpmd_internal)") From 92fddfcd2beeb47a714474c10ba3d393bb8f15ba Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 25 Apr 2024 19:17:08 +0200 Subject: [PATCH 093/121] Add two missing includes (found with iwyu) (#4895) * add missing include * add missing include --- Source/Diagnostics/OpenPMDHelpFunction.cpp | 3 +++ Source/Initialization/WarpXAMReXInit.cpp | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Source/Diagnostics/OpenPMDHelpFunction.cpp b/Source/Diagnostics/OpenPMDHelpFunction.cpp index 6170249b52b..cc798adc29e 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.cpp +++ b/Source/Diagnostics/OpenPMDHelpFunction.cpp @@ -6,8 +6,11 @@ * License: BSD-3-Clause-LBNL */ #include "OpenPMDHelpFunction.H" + #include "Utils/TextMsg.H" +#include + std::string WarpXOpenPMDFileType () { diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 8cfe78c740f..4036a49235c 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace { /** Overwrite defaults in AMReX Inputs * From 2aee638758948dca16ba832ed300e6adc28b61ae Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Apr 2024 06:30:22 -0700 Subject: [PATCH 094/121] `isort`: First Party Modules (#4902) * `isort`: First Party Modules Declare `amrex` and `pywarpx` our first party modules, even if they are not installed. This resolves inconsistencies when sorting imports in developer or remote runs (pre-commit.com CI). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/workflows/ml_materials/run_warpx_training.py | 1 + .../capacitive_discharge/PICMI_inputs_1d.py | 3 ++- .../capacitive_discharge/PICMI_inputs_2d.py | 3 ++- .../spacecraft_charging/PICMI_inputs_rz.py | 1 + .../Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py | 1 + Examples/Tests/langmuir/PICMI_inputs_rz.py | 1 + Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py | 1 + Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py | 1 + Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py | 1 + Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py | 1 + Examples/Tests/ohm_solver_EM_modes/analysis.py | 1 + Examples/Tests/ohm_solver_EM_modes/analysis_rz.py | 3 ++- Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py | 1 + Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py | 1 + Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py | 1 + Examples/Tests/ohm_solver_ion_beam_instability/analysis.py | 1 + .../Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py | 1 + .../Tests/particle_boundary_interaction/PICMI_inputs_rz.py | 1 + Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py | 1 + Examples/Tests/particle_data_python/PICMI_inputs_2d.py | 1 + .../Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py | 1 + Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py | 1 + Examples/Tests/python_wrappers/PICMI_inputs_2d.py | 1 + Examples/Tests/restart/PICMI_inputs_id_cpu_read.py | 1 + .../Tests/restart/PICMI_inputs_runtime_component_analyze.py | 1 + Python/pywarpx/picmi.py | 1 + pyproject.toml | 1 + 27 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py index 23fc49e7431..054c1b4cfc0 100644 --- a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -2,6 +2,7 @@ import math import numpy as np + from pywarpx import picmi # Physical constants diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 0b59516ccc7..8b58790d2e6 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -8,10 +8,11 @@ import sys import numpy as np -from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi + constants = picmi.constants diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py index 61229a17295..65baabba605 100755 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py @@ -6,10 +6,11 @@ # --- used for the field solve step. import numpy as np -from pywarpx import callbacks, fields, picmi from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla +from pywarpx import callbacks, fields, picmi + constants = picmi.constants ########################## diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 6eefdca942d..d541c2393ca 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -15,6 +15,7 @@ import numpy as np import scipy.constants as scc from mpi4py import MPI as mpi + from pywarpx import picmi from pywarpx.callbacks import installafterEsolve, installafterInitEsolve from pywarpx.fields import ExWrapper, EzWrapper, PhiFPWrapper, RhoFPWrapper diff --git a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py index 6104e08ef27..faec3ed4668 100755 --- a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py +++ b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import numpy as np + from pywarpx import fields, picmi max_steps = 1 diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 106b8de5a94..8328aba5185 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -9,6 +9,7 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np + from pywarpx import fields, picmi constants = picmi.constants diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py index f61823bf223..8f205724563 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py @@ -27,6 +27,7 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as con + from pywarpx import fields, picmi ########################## diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 6f7d6daa453..1268f7a02b0 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -27,6 +27,7 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as con + from pywarpx import fields, picmi ########################## diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 6711331d538..ec67282fd88 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -14,6 +14,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index e6f9ccc8e65..89c5230a6ad 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -12,6 +12,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis.py b/Examples/Tests/ohm_solver_EM_modes/analysis.py index 77be546937c..ad832dc2f50 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis.py @@ -6,6 +6,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index de7c9966b20..58e615c5332 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -8,10 +8,11 @@ import scipy.fft as fft from matplotlib import colors from openpmd_viewer import OpenPMDTimeSeries -from pywarpx import picmi from scipy.interpolate import RegularGridInterpolator from scipy.special import j1, jn, jn_zeros +from pywarpx import picmi + constants = picmi.constants # load simulation parameters diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index 8249bdb504f..d4430502f42 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -13,6 +13,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py index 93a4962d7e5..ade7187ebe3 100755 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py @@ -6,6 +6,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 186ef6cede3..4f8b5edcc3e 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -14,6 +14,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index 6388a4c4085..6a57b1c1046 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -7,6 +7,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index 5d4631e9c1e..56173893b7f 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -15,6 +15,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py index b9ea76e4a09..4017a05d413 100644 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py @@ -4,6 +4,7 @@ # --- This input is a simple case of reflection # --- of one electron on the surface of a sphere. import numpy as np + from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py index 63a805c2612..e5a9a58f597 100755 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py @@ -4,6 +4,7 @@ # --- to access the buffer of scraped particles. import numpy as np + from pywarpx import libwarpx, particle_containers, picmi ########################## diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index f43c2215b79..572871b8ed5 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -3,6 +3,7 @@ import sys import numpy as np + from pywarpx import callbacks, libwarpx, particle_containers, picmi # Create the parser and add the argument diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index 6f70b69fc10..5de9879f0f8 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -3,6 +3,7 @@ # --- Input file to test the saving of old particle positions import numpy as np + from pywarpx import particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py index 095702a1145..66f259da2ef 100755 --- a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py +++ b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py @@ -6,6 +6,7 @@ # --- if the correct amount of processors are initialized in AMReX. from mpi4py import MPI + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py index d01835f9fdf..db1cc7dcad8 100755 --- a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py +++ b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py @@ -3,6 +3,7 @@ import matplotlib.pyplot as plt import numpy as np from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable + from pywarpx import picmi # Number of time steps diff --git a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py index d76cef82fec..d400924a378 100755 --- a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py +++ b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py @@ -6,6 +6,7 @@ import sys import numpy as np + from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index daac6e1b5ac..32c9f4e5808 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -7,6 +7,7 @@ import sys import numpy as np + from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 2859d6fd468..902d6150773 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -14,6 +14,7 @@ import numpy as np import periodictable + import picmistandard import pywarpx diff --git a/pyproject.toml b/pyproject.toml index 5173216a0f4..f9e615f9b83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,5 @@ requires = [ build-backend = "setuptools.build_meta" [tool.isort] +known_first_party = ["amrex", "picmistandard", "pywarpx", "warpx"] profile = "black" From a294ff2fca63890d59d6613f41cb211d44aa72dc Mon Sep 17 00:00:00 2001 From: John Ryan Westernacher-Schneider Date: Mon, 29 Apr 2024 17:54:57 +0200 Subject: [PATCH 095/121] Clarify ParticleContainerWrapper.add_particles() docstring (#4903) --- Python/pywarpx/particle_containers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index f5330d05f84..70471e24d1a 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -42,17 +42,18 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, The type of species for which particles will be added x, y, z : arrays or scalars - The particle positions (default = 0.) + The particle positions (m) (default = 0.) ux, uy, uz : arrays or scalars - The particle momenta (default = 0.) + The particle proper velocities (m/s) (default = 0.) w : array or scalars Particle weights (default = 0.) unique_particles : bool - Whether the particles are unique or duplicated on several processes - (default = True) + True means the added particles are duplicated by each process; + False means the number of added particles is independent of + the number of processes (default = True) kwargs : dict Containing an entry for all the extra particle attribute arrays. If From 155336b513bbb455fec33624b84242df73f2eae3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Apr 2024 13:27:28 -0700 Subject: [PATCH 096/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4904) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index abd7e54c013..f73c39b6fc3 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - + cd ../amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 048efbc06be..94b334d02da 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = b752027c1aebdfb4be339b1e30932b4108286a7a +branch = d857968968ae2cdf18f39ee50df96529e326ff39 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 1f656960789..f8859ef7df5 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = b752027c1aebdfb4be339b1e30932b4108286a7a +branch = d857968968ae2cdf18f39ee50df96529e326ff39 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 022939dcbee..cbde4bca135 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "b752027c1aebdfb4be339b1e30932b4108286a7a" +set(WarpX_amrex_branch "d857968968ae2cdf18f39ee50df96529e326ff39" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 038b98c191b..fcf7c91bce2 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "28897d26f00707ee33d2423b8c24a020cd0767e0" +set(WarpX_pyamrex_branch "f370d38ab410dbf1b3a90683c97291841ebeb3f5" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 9bfc3ba5fbf..75e56e6964f 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - +cd amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 8dbd3bdfef2aa9b75e36a0c9ef5281994a5d549d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Apr 2024 16:48:49 -0700 Subject: [PATCH 097/121] Doc: Great Lakes ADIOS2 Update (#4905) Update the ADIOS2 v2.10.0 installation, do not create campaign directories (we do not use them yet). --- .../greatlakes-umich/install_v100_dependencies.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh index 7235b18c99f..9361fd6c64b 100755 --- a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -63,7 +63,15 @@ else git clone -b v2.10.0 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-v100-build -cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-v100-build -DADIOS2_USE_Blosc2=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 +cmake \ + -S $HOME/src/adios2 \ + -B ${build_dir}/adios2-v100-build \ + -DADIOS2_USE_Blosc2=ON \ + -DADIOS2_USE_Campaign=OFF \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 cmake --build ${build_dir}/adios2-v100-build --target install -j 8 rm -rf ${build_dir}/adios2-v100-build From c5a5732721b4e3e0b16a61a56fa29548109d9b9f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 30 Apr 2024 08:55:08 -0700 Subject: [PATCH 098/121] Fix external fields on grid in 1D (#4906) --- Source/Initialization/WarpXInitData.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e0c095918fa..ff2ca01deec 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1024,7 +1024,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1059,7 +1059,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1090,7 +1090,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; From 8a9bedc394a48b4c002d356be4e54b13a064356d Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 2 May 2024 14:43:50 -0700 Subject: [PATCH 099/121] Add option for Cartesian velocity grid with `VelocityCoincidenceThinning` (#4886) * initial implementation of the grid based merging routine * add compiler directives for different dimensions * avoid implicit capture of `this` in lambda * code clean-up * use `PIdx::x` for r-coordinate while 4667 is still being reviewed * fix clang-tidy errors * another clang-tidy fix * improve doc-strings * use iterative heap sort since "SYCL kernel cannot call a recursive function" * fix clang-tidy error * fix sorting bug; add documentation and CI test * add dummy CI benchmark file to get proper values * update benchmark values based on Azure results * rename `GridBasedMerging` -> `VelocityCoincidenceThinning` * use `Algorithms::KineticEnergy` * reorganize merging loop * update CI benchmark values * Relativistic correction in product particles' velocity calculation * update benchmark values after changing energy calculation * handle edge case with zero cluster momentum * call redistribute after particle resampling to remove invalid particles * use unsigned ints for indexing * Revert "use unsigned ints for indexing" This reverts commit abe027f3402cf0e7c51c4599a0a50b9e8a6b817a. * call `Redistribute` before merging * code clean-up * also check for `std::isnan` in edge case handling * add reference for grid based merging * check that cluster has total weight > 0 before merging * add defense against numerical error leading to nan * make resampling message more verbose * use `deleteInvalidParticles` instead of `Redistribute` * remove default values for merging parameters * remove doc-string default specifications in picmi.py * apply suggestions from code review * update benchmark values; avoid possible nans * add assert to prevent merging of photons * add `BackwardCompatibility` check to `LevelingThinning` * implement option for Cartesian velocity mesh * use `enum` for velocity grid types * fix Windows `uint` issue and clang-tidy error * use `Reduce` functions for GPU compatibility * merge kernels finding min and max velocities * take array of du values in picmi * avoid checking velocity grid type for every particle * fix issue in picmi * use array input for velocity bin sizes --- Python/pywarpx/picmi.py | 24 +++- .../Resampling/VelocityCoincidenceThinning.H | 95 ++++++++++++++- .../VelocityCoincidenceThinning.cpp | 113 ++++++++++++------ 3 files changed, 187 insertions(+), 45 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 902d6150773..03f2ed14732 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -147,17 +147,27 @@ class Species(picmistandard.PICMI_Species): warpx_resampling_algorithm: str, default="leveling_thinning" Resampling algorithm to use. + warpx_resampling_algorithm_velocity_grid_type: str, default="spherical" + Type of grid to use when clustering particles in velocity space. Only + applicable with the `velocity_coincidence_thinning` algorithm. + warpx_resampling_algorithm_delta_ur: float Size of velocity window used for clustering particles during grid-based - merging. + merging, with `velocity_grid_type == "spherical"`. warpx_resampling_algorithm_n_theta: int Number of bins to use in theta when clustering particle velocities - during grid-based merging. + during grid-based merging, with `velocity_grid_type == "spherical"`. warpx_resampling_algorithm_n_phi: int Number of bins to use in phi when clustering particle velocities - during grid-based merging. + during grid-based merging, with `velocity_grid_type == "spherical"`. + + warpx_resampling_algorithm_delta_u: array of floats or float + Size of velocity window used in ux, uy and uz for clustering particles + during grid-based merging, with `velocity_grid_type == "cartesian"`. If + a single number is given the same du value will be used in all three + directions. """ def init(self, kw): @@ -239,9 +249,13 @@ def init(self, kw): self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) + self.resampling_algorithm_delta_u = kw.pop('warpx_resampling_algorithm_delta_u', None) + if np.size(self.resampling_algorithm_delta_u) == 1: + self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 def species_initialize_inputs(self, layout, initialize_self_fields = False, @@ -284,9 +298,11 @@ def species_initialize_inputs(self, layout, resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, - resampling_algorithm_n_phi=self.resampling_algorithm_n_phi) + resampling_algorithm_n_phi=self.resampling_algorithm_n_phi, + resampling_algorithm_delta_u=self.resampling_algorithm_delta_u) # add reflection models self.species.add_new_attr("reflection_model_xlo(E)", self.reflection_model_xlo) diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H index 9462f4b35ed..b8d67d198a4 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.H +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -9,8 +9,10 @@ #ifndef WARPX_VELOCITY_COINCIDENCE_THINNING_H_ #define WARPX_VELOCITY_COINCIDENCE_THINNING_H_ +#include "Particles/Algorithms/KineticEnergy.H" #include "Resampling.H" - +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" /** * \brief This class implements a particle merging scheme wherein particles @@ -33,6 +35,11 @@ public: */ VelocityCoincidenceThinning (const std::string& species_name); + enum struct VelocityGridType { + Spherical = 0, + Cartesian = 1 + }; + /** * \brief A method that performs merging for the considered species. * @@ -106,10 +113,92 @@ public: } }; + /** + * \brief Struct used to assign velocity space bin numbers to a given set + * of particles. + */ + struct VelocityBinCalculator { + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void labelOnSphericalVelocityGrid (const amrex::ParticleReal ux[], + const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], + const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop ) const + { + for (int i = cell_start; i < cell_stop; ++i) + { + // get polar components of the velocity vector + auto u_mag = std::sqrt( + ux[indices[i]]*ux[indices[i]] + + uy[indices[i]]*uy[indices[i]] + + uz[indices[i]]*uz[indices[i]] + ); + auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; + auto u_phi = std::acos(uz[indices[i]]/u_mag); + + const int ii = static_cast(u_theta / dutheta); + const int jj = static_cast(u_phi / duphi); + const int kk = static_cast(u_mag / dur); + + bin_array[i] = ii + jj * n1 + kk * n1 * n2; + index_array[i] = i; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void labelOnCartesianVelocityGrid (const amrex::ParticleReal ux[], + const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], + const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop ) const + { + for (int i = cell_start; i < cell_stop; ++i) + { + const int ii = static_cast((ux[indices[i]] - ux_min) / dux); + const int jj = static_cast((uy[indices[i]] - uy_min) / duy); + const int kk = static_cast((uz[indices[i]] - uz_min) / duz); + + bin_array[i] = ii + jj * n1 + kk * n1 * n2; + index_array[i] = i; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator() (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop) const + { + if (velocity_grid_type == VelocityGridType::Spherical) { + labelOnSphericalVelocityGrid( + ux, uy, uz, indices, bin_array, index_array, cell_start, + cell_stop + ); + } + else if (velocity_grid_type == VelocityGridType::Cartesian) { + labelOnCartesianVelocityGrid( + ux, uy, uz, indices, bin_array, index_array, cell_start, + cell_stop + ); + } + } + + VelocityGridType velocity_grid_type; + int n1, n2; + amrex::ParticleReal dur, dutheta, duphi; + amrex::ParticleReal dux, duy, duz; + amrex::ParticleReal ux_min, uy_min, uz_min, ux_max, uy_max; + }; + private: + VelocityGridType m_velocity_grid_type; + int m_min_ppc = 1; - int m_ntheta; - int m_nphi; + int m_ntheta, m_nphi; amrex::ParticleReal m_delta_ur; + amrex::Vector m_delta_u; }; #endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp index 66176c48b4d..db4ddb801c6 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -9,10 +9,6 @@ #include "VelocityCoincidenceThinning.H" -#include "Particles/Algorithms/KineticEnergy.H" -#include "Utils/Parser/ParserUtils.H" -#include "Utils/ParticleUtils.H" - VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& species_name) { @@ -28,15 +24,33 @@ VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& spe "Resampling min_ppc should be greater than or equal to 1" ); - utils::parser::getWithParser( - pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur - ); - utils::parser::getWithParser( - pp_species_name, "resampling_algorithm_n_theta", m_ntheta - ); - utils::parser::getWithParser( - pp_species_name, "resampling_algorithm_n_phi", m_nphi + std::string velocity_grid_type_str = "spherical"; + pp_species_name.query( + "resampling_algorithm_velocity_grid_type", velocity_grid_type_str ); + if (velocity_grid_type_str == "spherical") { + m_velocity_grid_type = VelocityGridType::Spherical; + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_theta", m_ntheta + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_phi", m_nphi + ); + } + else if (velocity_grid_type_str == "cartesian") { + m_velocity_grid_type = VelocityGridType::Cartesian; + utils::parser::getArrWithParser( + pp_species_name, "resampling_algorithm_delta_u", m_delta_u + ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_delta_u.size() == 3, + "resampling_algorithm_delta_u must have three components."); + } + else { + WARPX_ABORT_WITH_MESSAGE("Unkown velocity grid type."); + } } void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, @@ -45,6 +59,7 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, using namespace amrex::literals; auto& ptile = pc->ParticlesAt(lev, pti); + const auto n_parts_in_tile = pti.numParticles(); auto& soa = ptile.GetStructOfArrays(); #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); @@ -79,21 +94,57 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, ); // create a GPU vector to hold the momentum cluster index for each particle - amrex::Gpu::DeviceVector momentum_bin_number(bins.numItems()); + amrex::Gpu::DeviceVector momentum_bin_number(n_parts_in_tile); auto* momentum_bin_number_data = momentum_bin_number.data(); // create a GPU vector to hold the index sorting for the momentum bins - amrex::Gpu::DeviceVector sorted_indices(bins.numItems()); + amrex::Gpu::DeviceVector sorted_indices(n_parts_in_tile); auto* sorted_indices_data = sorted_indices.data(); - const auto Ntheta = m_ntheta; - const auto Nphi = m_nphi; - - const auto dr = m_delta_ur; - const auto dtheta = 2.0_prt * MathConst::pi / Ntheta; - const auto dphi = MathConst::pi / Nphi; constexpr auto c2 = PhysConst::c * PhysConst::c; + auto velocityBinCalculator = VelocityBinCalculator(); + velocityBinCalculator.velocity_grid_type = m_velocity_grid_type; + if (m_velocity_grid_type == VelocityGridType::Spherical) { + velocityBinCalculator.dur = m_delta_ur; + velocityBinCalculator.n1 = m_ntheta; + velocityBinCalculator.n2 = m_nphi; + velocityBinCalculator.dutheta = 2.0_prt * MathConst::pi / m_ntheta; + velocityBinCalculator.duphi = MathConst::pi / m_nphi; + } + else if (m_velocity_grid_type == VelocityGridType::Cartesian) { + velocityBinCalculator.dux = m_delta_u[0]; + velocityBinCalculator.duy = m_delta_u[1]; + velocityBinCalculator.duz = m_delta_u[2]; + + // get the minimum and maximum velocities to determine the velocity space + // grid boundaries + { + using ReduceOpsT = amrex::TypeMultiplier; + using ReduceDataT = amrex::TypeMultiplier; + ReduceOpsT reduce_op; + ReduceDataT reduce_data(reduce_op); + using ReduceTuple = typename ReduceDataT::Type; + reduce_op.eval(n_parts_in_tile, reduce_data, [=] AMREX_GPU_DEVICE(int i) -> ReduceTuple { + return {ux[i], uy[i], uz[i], ux[i], uy[i]}; + }); + auto hv = reduce_data.value(reduce_op); + velocityBinCalculator.ux_min = amrex::get<0>(hv); + velocityBinCalculator.uy_min = amrex::get<1>(hv); + velocityBinCalculator.uz_min = amrex::get<2>(hv); + velocityBinCalculator.ux_max = amrex::get<3>(hv); + velocityBinCalculator.uy_max = amrex::get<4>(hv); + } + + velocityBinCalculator.n1 = static_cast( + std::ceil((velocityBinCalculator.ux_max - velocityBinCalculator.ux_min) / m_delta_u[0]) + ); + velocityBinCalculator.n2 = static_cast( + std::ceil((velocityBinCalculator.uy_max - velocityBinCalculator.uy_min) / m_delta_u[1]) + ); + } auto heapSort = HeapSort(); // Loop over cells @@ -114,24 +165,10 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, // Loop over particles and label them with the appropriate momentum bin // number. Also assign initial ordering to the sorted_indices array. - for (int i = cell_start; i < cell_stop; ++i) - { - // get polar components of the velocity vector - auto u_mag = std::sqrt( - ux[indices[i]]*ux[indices[i]] + - uy[indices[i]]*uy[indices[i]] + - uz[indices[i]]*uz[indices[i]] - ); - auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; - auto u_phi = std::acos(uz[indices[i]]/u_mag); - - const auto ii = static_cast(u_theta / dtheta); - const auto jj = static_cast(u_phi / dphi); - const auto kk = static_cast(u_mag / dr); - - momentum_bin_number_data[i] = ii + jj * Ntheta + kk * Ntheta * Nphi; - sorted_indices_data[i] = i; - } + velocityBinCalculator( + ux, uy, uz, indices, momentum_bin_number_data, sorted_indices_data, + cell_start, cell_stop + ); // sort indices based on comparing values in momentum_bin_number heapSort(sorted_indices_data, momentum_bin_number_data, cell_start, cell_numparts); From bc5d0a2cb39b375ac23ec99146bd241bdef62f7f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 3 May 2024 10:09:08 -0700 Subject: [PATCH 100/121] Fix Python installation instructions on Perlmutter (#4907) --- Docs/source/install/hpc/perlmutter.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index e9ebcd4e1de..14972566abc 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -76,7 +76,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx/bin/activate + source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate .. dropdown:: Script Details :color: light @@ -126,7 +126,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx/bin/activate + source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate .. dropdown:: Script Details :color: light From 3f413a3ee4e777780feefe619cb1dd49adf788f0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 6 May 2024 22:30:21 -0700 Subject: [PATCH 101/121] Docs: Fix PASC Paper Formatting --- Docs/source/highlights.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 91c99ee50ca..c94662d531f 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -27,7 +27,7 @@ Scientific works in laser-plasma and beam-plasma acceleration. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. @@ -104,7 +104,7 @@ Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. From 0e2fc85fd0de1e5357d84c362dd0f275b7278f50 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 7 May 2024 10:17:24 -0700 Subject: [PATCH 102/121] Highlights: Fix Formatting Issue Render issue in RST in PASC24 entry. --- Docs/source/highlights.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index c94662d531f..b243f62cd97 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -25,9 +25,9 @@ Scientific works in laser-plasma and beam-plasma acceleration. `DOI:10.1103/PhysRevResearch.5.033112 `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. @@ -102,9 +102,9 @@ Particle Accelerator & Beam Physics Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. From 4a6ccdce898889fbff2c4df36cd7d270b371953f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 9 May 2024 01:17:17 +0200 Subject: [PATCH 103/121] make do_current_centering a regular member variable of the WarpX class (#4901) --- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/Parallelization/WarpXComm.cpp | 4 ++-- Source/WarpX.H | 2 +- Source/WarpX.cpp | 2 -- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 51fcb5c262c..de586895c2d 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -645,7 +645,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (dt[0] denotes the time step on mesh refinement level 0) if (J_in_time == JInTime::Linear) { - auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; + auto& current = (do_current_centering) ? current_fp_nodal : current_fp; mypc->DepositCurrent(current, dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', @@ -679,7 +679,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) - auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; + auto& current = (do_current_centering) ? current_fp_nodal : current_fp; mypc->DepositCurrent(current, dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 9daf37af3f8..f3bff69546b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -882,7 +882,7 @@ WarpX::SyncCurrent ( WARPX_PROFILE("WarpX::SyncCurrent()"); // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid - if (WarpX::do_current_centering) + if (do_current_centering) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); @@ -1178,7 +1178,7 @@ void WarpX::SumBoundaryJ ( const amrex::IntVect ng = J.nGrowVect(); amrex::IntVect ng_depos_J = get_ng_depos_J(); - if (WarpX::do_current_centering) + if (do_current_centering) { #if defined(WARPX_DIM_1D_Z) ng_depos_J[0] += WarpX::current_centering_noz / 2; diff --git a/Source/WarpX.H b/Source/WarpX.H index d19843ca636..6c6538325e6 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -211,7 +211,7 @@ public: //! If true, the current is deposited on a nodal grid and then centered onto a staggered grid //! using finite centering of order given by #current_centering_nox, #current_centering_noy, //! and #current_centering_noz - static bool do_current_centering; + bool do_current_centering = false; //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9a29d7c2354..8c3a6f15b1c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -143,8 +143,6 @@ amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,FieldBo amrex::Vector WarpX::particle_boundary_lo(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); amrex::Vector WarpX::particle_boundary_hi(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); -bool WarpX::do_current_centering = false; - int WarpX::n_rz_azimuthal_modes = 1; int WarpX::ncomps = 1; From 98bd725860fad3d69cc21a6f9d60548ef9fe43ca Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 9 May 2024 21:38:16 -0700 Subject: [PATCH 104/121] Add CI test for velocity coincidence thinning (#4919) * add CI test for Cartesian velocity grid in resampling * avoid unneeded picmi input parameters * set checksum values for new test * use seperate input file for new example --- ...1d_velocity_coincidence_thinning_cartesian | 45 +++++++++++++++++++ Python/pywarpx/picmi.py | 2 +- ...locity_coincidence_thinning_cartesian.json | 20 +++++++++ Regression/WarpX-tests.ini | 17 +++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian create mode 100644 Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian new file mode 100644 index 00000000000..97830d8bb9e --- /dev/null +++ b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian @@ -0,0 +1,45 @@ +max_step = 4 +warpx.verbose = 1 +warpx.const_dt = 1e-10 +amr.n_cell = 256 +amr.max_grid_size = 64 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = 0 +geometry.prob_hi = 0.1 + +# Boundary condition and field solver +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic +algo.particle_shape = 1 +algo.maxwell_solver = none + +particles.species_names = hydrogen +hydrogen.mass = 1.67262192369e-27 +hydrogen.charge = 1.602176634e-19 +hydrogen.injection_style = nrandompercell +hydrogen.initialize_self_fields = 0 +hydrogen.do_not_push = 1 +hydrogen.do_resampling = 1 +hydrogen.resampling_min_ppc = 10 +hydrogen.resampling_trigger_intervals = 1::2 +hydrogen.resampling_algorithm = velocity_coincidence_thinning +hydrogen.resampling_algorithm_velocity_grid_type = cartesian +hydrogen.resampling_algorithm_delta_u = 10000000.0 5000000.0 7000000.0 +hydrogen.num_particles_per_cell = 2500 +hydrogen.momentum_distribution_type = gaussian +hydrogen.ux_m = 0.0 +hydrogen.uy_m = 0.0 +hydrogen.uz_m = 0.0 +hydrogen.ux_th = 0.000326 +hydrogen.uy_th = 0.000326 +hydrogen.uz_th = 0.000326 +hydrogen.profile = constant +hydrogen.density = 1e+19 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 4 +diag1.diag_type = Full diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 03f2ed14732..e7575f83b26 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -254,7 +254,7 @@ def init(self, kw): self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) self.resampling_algorithm_delta_u = kw.pop('warpx_resampling_algorithm_delta_u', None) - if np.size(self.resampling_algorithm_delta_u) == 1: + if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 def species_initialize_inputs(self, layout, diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json new file mode 100644 index 00000000000..9f1dae16e7f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 434240.88525516074, + "jy": 315768.9402331213, + "jz": 642846.5589763735 + }, + "hydrogen": { + "particle_momentum_x": 6.919253235905925e-20, + "particle_momentum_y": 7.069867215063566e-20, + "particle_momentum_z": 7.606884436016626e-20, + "particle_position_x": 25.60006676227001, + "particle_weight": 1.0000000000000001e+18 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f8859ef7df5..edbd97b0acb 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4206,6 +4206,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py +[resample_velocity_coincidence_thinning_cartesian] +buildDir = . +inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/analysis_default_regression.py + [restart] buildDir = . inputFile = Examples/Tests/restart/inputs From fee6c4e204add92d3676b715343943ec7a694a9b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 10 May 2024 10:09:24 -0700 Subject: [PATCH 105/121] Amend Notices for Non-Endorsement (#4924) Amend and move NOTICES to NOTICE.txt. Appended Text: Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government, The Regents of the University of California, or Lawrence Berkeley National Laboratory. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government, The Regents of the University of California, or Lawrence Berkeley National Laboratory, and shall not be used for advertising or product endorsement purposes. --- NOTICE.txt | 19 +++++++++++++++++++ README.md | 9 +-------- 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 NOTICE.txt diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000000..e556dddec6c --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,19 @@ +This Software was developed under funding from the U.S. Department +of Energy and the U.S. Government consequently retains certain rights. As +such, the U.S. Government has been granted for itself and others acting on +its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the +Software to reproduce, distribute copies to the public, prepare derivative +works, and perform publicly and display publicly, and to permit other to do +so. + +Reference herein to any specific commercial product, process, or service +by trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the +United States Government, The Regents of the University of California, or +Lawrence Berkeley National Laboratory. + +The views and opinions of authors expressed herein do not necessarily +state or reflect those of the United States Government, +The Regents of the University of California, or +Lawrence Berkeley National Laboratory, and shall not be used for advertising +or product endorsement purposes. diff --git a/README.md b/README.md index 87cf2b012c0..b2de2211b60 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,6 @@ If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. -NOTICE. This Software was developed under funding from the U.S. Department -of Energy and the U.S. Government consequently retains certain rights. As -such, the U.S. Government has been granted for itself and others acting on -its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the -Software to reproduce, distribute copies to the public, prepare derivative -works, and perform publicly and display publicly, and to permit other to do -so. - Please see the full license agreement in [LICENSE.txt](LICENSE.txt). +Please see the notices in [NOTICE.txt](NOTICE.txt). The SPDX license identifier is `BSD-3-Clause-LBNL`. From e6c1eb5289f39e34fc4093074d0439f7fa395abf Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 12 May 2024 12:16:45 -0700 Subject: [PATCH 106/121] Diagnostics: Fix Position Selection (#4914) * Diagnostics: Warn w/o Positions If a user specifies `..variables` without including particle position arguments we now warn. * openPMD: Skip 0-positions for deselected comps * Clang-Tidy: Braces * Update Examples * RZ: x,y,z * Fix: openPMD Low-D position/positionOffset Continue to write defaults for coarse and fine position of omitted dimensions to zero. --- Docs/source/usage/parameters.rst | 5 +- .../laser_acceleration/inputs_rz | 4 +- .../uniform_plasma/inputs_3d | 2 +- Examples/Tests/Implicit/inputs_1d | 4 +- .../Tests/Implicit/inputs_1d_semiimplicit | 4 +- Examples/Tests/Implicit/inputs_vandb_2d | 4 +- Examples/Tests/electrostatic_sphere/inputs_rz | 2 +- Examples/Tests/langmuir/inputs_3d | 4 +- Examples/Tests/photon_pusher/inputs_3d | 32 ++++---- Examples/Tests/plasma_lens/inputs_3d | 2 +- Examples/Tests/plasma_lens/inputs_boosted_3d | 2 +- Examples/Tests/plasma_lens/inputs_lattice_3d | 2 +- Examples/Tests/plasma_lens/inputs_short_3d | 2 +- Examples/Tests/qed/breit_wheeler/inputs_2d | 28 +++---- Examples/Tests/qed/breit_wheeler/inputs_3d | 28 +++---- .../Tests/qed/quantum_synchrotron/inputs_2d | 18 ++--- .../Tests/qed/quantum_synchrotron/inputs_3d | 18 ++--- Regression/WarpX-tests.ini | 40 +++++----- .../Diagnostics/ParticleDiag/ParticleDiag.cpp | 25 ++++++- Source/Diagnostics/WarpXOpenPMD.H | 6 +- Source/Diagnostics/WarpXOpenPMD.cpp | 73 +++++++++++-------- 21 files changed, 169 insertions(+), 136 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 5f97730e880..40c16a06df5 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2689,10 +2689,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``..variables`` (list of `strings` separated by spaces, optional) List of particle quantities to write to output. - Choices are ``w`` for the particle weight and ``ux`` ``uy`` ``uz`` for the particle momenta. + Choices are ``x``, ``y``, ``z`` for the particle positions (3D and RZ), ``x`` & ``z`` in 2D, ``z`` in 1D, + ``w`` for the particle weight and ``ux``, ``uy``, ``uz`` for the particle momenta. When using the lab-frame electrostatic solver, ``phi`` (electrostatic potential, on the macroparticles) is also available. By default, all particle quantities (except ``phi``) are written. - If ``..variables = none``, no particle data are written, except for particle positions, which are always included. + If ``..variables = none``, no particle data are written. * ``..random_fraction`` (`float`) optional If provided ``..random_fraction = a``, only `a` fraction of the particle data of this species will be dumped randomly in diag ````, i.e. if `rand() < a`, this particle will be dumped, where `rand()` denotes a random number generator. diff --git a/Examples/Physics_applications/laser_acceleration/inputs_rz b/Examples/Physics_applications/laser_acceleration/inputs_rz index 7365523b574..b61ed6f83ec 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_rz +++ b/Examples/Physics_applications/laser_acceleration/inputs_rz @@ -89,5 +89,5 @@ diagnostics.diags_names = diag1 diag1.intervals = 10 diag1.diag_type = Full diag1.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz rho -diag1.electrons.variables = w ux uy uz orig_x orig_z -diag1.beam.variables = w ux uy uz +diag1.electrons.variables = x y z w ux uy uz orig_x orig_z +diag1.beam.variables = x y z w ux uy uz diff --git a/Examples/Physics_applications/uniform_plasma/inputs_3d b/Examples/Physics_applications/uniform_plasma/inputs_3d index b0275f6eed6..83310088176 100644 --- a/Examples/Physics_applications/uniform_plasma/inputs_3d +++ b/Examples/Physics_applications/uniform_plasma/inputs_3d @@ -45,7 +45,7 @@ electrons.uz_th = 0.01 # uth the std of the (unitless) momentum diagnostics.diags_names = diag1 chk diag1.intervals = 4 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz w +diag1.electrons.variables = x y z ux uy uz w diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho chk.intervals = 6 diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d index 50d28a2db75..465d9dd6965 100644 --- a/Examples/Tests/Implicit/inputs_1d +++ b/Examples/Tests/Implicit/inputs_1d @@ -71,8 +71,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit index 2271a0bb1bc..4008a559588 100644 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -71,8 +71,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_2d index 2dc57323efe..33ce964710a 100644 --- a/Examples/Tests/Implicit/inputs_vandb_2d +++ b/Examples/Tests/Implicit/inputs_vandb_2d @@ -82,8 +82,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 20 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/electrostatic_sphere/inputs_rz b/Examples/Tests/electrostatic_sphere/inputs_rz index 26d8b0c7afc..2b6151e6d8c 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_rz +++ b/Examples/Tests/electrostatic_sphere/inputs_rz @@ -39,5 +39,5 @@ diag1.fields_to_plot = Er Et Ez rho diag2.intervals = 30 diag2.diag_type = Full diag2.fields_to_plot = none -diag2.electron.variables = ux uy uz w phi +diag2.electron.variables = x y z ux uy uz w phi diag2.format = openpmd diff --git a/Examples/Tests/langmuir/inputs_3d b/Examples/Tests/langmuir/inputs_3d index 432b4788171..604ed9bc514 100644 --- a/Examples/Tests/langmuir/inputs_3d +++ b/Examples/Tests/langmuir/inputs_3d @@ -93,5 +93,5 @@ diagnostics.diags_names = diag1 diag1.intervals = max_step diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho -diag1.electrons.variables = w ux -diag1.positrons.variables = uz +diag1.electrons.variables = x y z w ux +diag1.positrons.variables = x y z uz diff --git a/Examples/Tests/photon_pusher/inputs_3d b/Examples/Tests/photon_pusher/inputs_3d index e2c75981e0e..3977a187e1b 100644 --- a/Examples/Tests/photon_pusher/inputs_3d +++ b/Examples/Tests/photon_pusher/inputs_3d @@ -140,19 +140,19 @@ p_dn_10.single_particle_weight = 1.0 diagnostics.diags_names = diag1 diag1.intervals = 50 diag1.diag_type = Full -diag1.p_xp_1.variables = ux uy uz -diag1.p_xn_1.variables = ux uy uz -diag1.p_yp_1.variables = ux uy uz -diag1.p_yn_1.variables = ux uy uz -diag1.p_zp_1.variables = ux uy uz -diag1.p_zn_1.variables = ux uy uz -diag1.p_dp_1.variables = ux uy uz -diag1.p_dn_1.variables = ux uy uz -diag1.p_xp_10.variables = ux uy uz -diag1.p_xn_10.variables = ux uy uz -diag1.p_yp_10.variables = ux uy uz -diag1.p_yn_10.variables = ux uy uz -diag1.p_zp_10.variables = ux uy uz -diag1.p_zn_10.variables = ux uy uz -diag1.p_dp_10.variables = ux uy uz -diag1.p_dn_10.variables = ux uy uz +diag1.p_xp_1.variables = x y z ux uy uz +diag1.p_xn_1.variables = x y z ux uy uz +diag1.p_yp_1.variables = x y z ux uy uz +diag1.p_yn_1.variables = x y z ux uy uz +diag1.p_zp_1.variables = x y z ux uy uz +diag1.p_zn_1.variables = x y z ux uy uz +diag1.p_dp_1.variables = x y z ux uy uz +diag1.p_dn_1.variables = x y z ux uy uz +diag1.p_xp_10.variables = x y z ux uy uz +diag1.p_xn_10.variables = x y z ux uy uz +diag1.p_yp_10.variables = x y z ux uy uz +diag1.p_yn_10.variables = x y z ux uy uz +diag1.p_zp_10.variables = x y z ux uy uz +diag1.p_zn_10.variables = x y z ux uy uz +diag1.p_dp_10.variables = x y z ux uy uz +diag1.p_dn_10.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_3d b/Examples/Tests/plasma_lens/inputs_3d index 7763013adb1..a87c1eb7f44 100644 --- a/Examples/Tests/plasma_lens/inputs_3d +++ b/Examples/Tests/plasma_lens/inputs_3d @@ -48,4 +48,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_boosted_3d b/Examples/Tests/plasma_lens/inputs_boosted_3d index ecdf3b3baec..fa18ac439c4 100644 --- a/Examples/Tests/plasma_lens/inputs_boosted_3d +++ b/Examples/Tests/plasma_lens/inputs_boosted_3d @@ -50,4 +50,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_lattice_3d b/Examples/Tests/plasma_lens/inputs_lattice_3d index a41ca8baee9..94e57de8290 100644 --- a/Examples/Tests/plasma_lens/inputs_lattice_3d +++ b/Examples/Tests/plasma_lens/inputs_lattice_3d @@ -70,4 +70,4 @@ plasmalens4.dEdx = 200000. diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_short_3d b/Examples/Tests/plasma_lens/inputs_short_3d index 8fa111df3c9..5c91ab26498 100644 --- a/Examples/Tests/plasma_lens/inputs_short_3d +++ b/Examples/Tests/plasma_lens/inputs_short_3d @@ -52,4 +52,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 1 #84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/qed/breit_wheeler/inputs_2d b/Examples/Tests/qed/breit_wheeler/inputs_2d index 7f507dd08d7..857b3243ac6 100644 --- a/Examples/Tests/qed/breit_wheeler/inputs_2d +++ b/Examples/Tests/qed/breit_wheeler/inputs_2d @@ -201,19 +201,19 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthBW -diag1.p2.variables = ux uy uz w opticalDepthBW -diag1.p3.variables = ux uy uz w opticalDepthBW -diag1.p4.variables = ux uy uz w opticalDepthBW - -diag1.ele1.variables = ux uy uz w opticalDepthQSR -diag1.ele2.variables = ux uy uz w opticalDepthQSR -diag1.ele3.variables = ux uy uz w opticalDepthQSR -diag1.ele4.variables = ux uy uz w opticalDepthQSR - -diag1.pos1.variables = ux uy uz w opticalDepthQSR -diag1.pos2.variables = ux uy uz w opticalDepthQSR -diag1.pos3.variables = ux uy uz w opticalDepthQSR -diag1.pos4.variables = ux uy uz w opticalDepthQSR +diag1.p1.variables = x z ux uy uz w opticalDepthBW +diag1.p2.variables = x z ux uy uz w opticalDepthBW +diag1.p3.variables = x z ux uy uz w opticalDepthBW +diag1.p4.variables = x z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x z ux uy uz w opticalDepthQSR diag1.format = plotfile diff --git a/Examples/Tests/qed/breit_wheeler/inputs_3d b/Examples/Tests/qed/breit_wheeler/inputs_3d index 2749a60b888..b79dfd9c030 100644 --- a/Examples/Tests/qed/breit_wheeler/inputs_3d +++ b/Examples/Tests/qed/breit_wheeler/inputs_3d @@ -201,19 +201,19 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthBW -diag1.p2.variables = ux uy uz w opticalDepthBW -diag1.p3.variables = ux uy uz w opticalDepthBW -diag1.p4.variables = ux uy uz w opticalDepthBW - -diag1.ele1.variables = ux uy uz w opticalDepthQSR -diag1.ele2.variables = ux uy uz w opticalDepthQSR -diag1.ele3.variables = ux uy uz w opticalDepthQSR -diag1.ele4.variables = ux uy uz w opticalDepthQSR - -diag1.pos1.variables = ux uy uz w opticalDepthQSR -diag1.pos2.variables = ux uy uz w opticalDepthQSR -diag1.pos3.variables = ux uy uz w opticalDepthQSR -diag1.pos4.variables = ux uy uz w opticalDepthQSR +diag1.p1.variables = x y z ux uy uz w opticalDepthBW +diag1.p2.variables = x y z ux uy uz w opticalDepthBW +diag1.p3.variables = x y z ux uy uz w opticalDepthBW +diag1.p4.variables = x y z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x y z ux uy uz w opticalDepthQSR diag1.format = plotfile diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_2d b/Examples/Tests/qed/quantum_synchrotron/inputs_2d index 2fc00231280..2ac2c782ccd 100644 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_2d +++ b/Examples/Tests/qed/quantum_synchrotron/inputs_2d @@ -178,12 +178,12 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthQSR -diag1.p2.variables = ux uy uz w opticalDepthQSR -diag1.p3.variables = ux uy uz w opticalDepthQSR -diag1.p4.variables = ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = ux uy uz w opticalDepthBW -diag1.qsp_2.variables = ux uy uz w opticalDepthBW -diag1.qsp_3.variables = ux uy uz w opticalDepthBW -diag1.qsp_4.variables = ux uy uz w opticalDepthBW +diag1.p1.variables = x z ux uy uz w opticalDepthQSR +diag1.p2.variables = x z ux uy uz w opticalDepthQSR +diag1.p3.variables = x z ux uy uz w opticalDepthQSR +diag1.p4.variables = x z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_3d b/Examples/Tests/qed/quantum_synchrotron/inputs_3d index 9bc963f7e94..429666ef938 100644 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_3d +++ b/Examples/Tests/qed/quantum_synchrotron/inputs_3d @@ -178,12 +178,12 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthQSR -diag1.p2.variables = ux uy uz w opticalDepthQSR -diag1.p3.variables = ux uy uz w opticalDepthQSR -diag1.p4.variables = ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = ux uy uz w opticalDepthBW -diag1.qsp_2.variables = ux uy uz w opticalDepthBW -diag1.qsp_3.variables = ux uy uz w opticalDepthBW -diag1.qsp_4.variables = ux uy uz w opticalDepthBW +diag1.p1.variables = x y z ux uy uz w opticalDepthQSR +diag1.p2.variables = x y z ux uy uz w opticalDepthQSR +diag1.p3.variables = x y z ux uy uz w opticalDepthQSR +diag1.p4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x y z ux uy uz w opticalDepthBW diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index edbd97b0acb..d31a042c807 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -554,7 +554,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w phi +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w phi dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -1189,7 +1189,7 @@ analysisOutputImage = langmuir_fluid_multi_analysis.png [Langmuir_multi_1d] buildDir = . inputFile = Examples/Tests/langmuir/inputs_1d -runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=z w ux uy uz diag1.positrons.variables=z w ux uy uz dim = 1 addToCompileString = USE_OPENPMD=TRUE QED=FALSE cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF @@ -1208,7 +1208,7 @@ analysisOutputImage = langmuir_multi_1d_analysis.png [Langmuir_multi_2d_MR] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz +runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1227,7 +1227,7 @@ analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_anisotropic] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz +runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1246,7 +1246,7 @@ analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1265,7 +1265,7 @@ analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png [Langmuir_multi_2d_MR_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1284,7 +1284,7 @@ analysisOutputImage = Langmuir_multi_2d_MR_psatd.png [Langmuir_multi_2d_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1303,7 +1303,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1322,7 +1322,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1341,7 +1341,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1360,7 +1360,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1417,7 +1417,7 @@ analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png [Langmuir_multi_2d_psatd_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1436,7 +1436,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1455,7 +1455,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1474,7 +1474,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1492,7 +1492,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d +inputFile = runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = @@ -1721,7 +1721,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_rz] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 +runtime_params = diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ @@ -1741,7 +1741,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1761,7 +1761,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1781,7 +1781,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_multiJ] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 +runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 7ed64ccf8ec..1a64ae20f0e 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -6,6 +6,8 @@ #include "Utils/TextMsg.H" #include "WarpX.H" +#include + #include #include @@ -13,7 +15,7 @@ using namespace amrex; -ParticleDiag::ParticleDiag( +ParticleDiag::ParticleDiag ( const std::string& diag_name, const std::string& name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc): m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) @@ -28,12 +30,17 @@ ParticleDiag::ParticleDiag( amrex::Vector variables; const int variables_specified = pp_diag_name_species_name.queryarr("variables", variables); - if (variables_specified){ + if (variables_specified) { // If only specific variables have been specified, fill m_plot_flags with zero and only set // requested variables to one std::fill(m_plot_flags.begin(), m_plot_flags.end(), 0); + bool contains_positions = false; if (variables[0] != "none"){ - const std::map existing_variable_names = pc->getParticleComps(); + std::map existing_variable_names = pc->getParticleComps(); +#ifdef WARPX_DIM_RZ + // we reconstruct to Cartesian x,y,z for RZ particle output + existing_variable_names["y"] = PIdx::theta; +#endif for (const auto& var : variables){ if (var == "phi") { // User requests phi on particle. This is *not* part of the variables that @@ -47,9 +54,21 @@ ParticleDiag::ParticleDiag( "variables argument '" + var +"' is not an existing attribute for this species"); m_plot_flags[existing_variable_names.at(var)] = 1; + + if (var == "x" || var == "y" || var == "z") { + contains_positions = true; + } } } } + + if (!contains_positions) { + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + diag_name + "." + name + ".variables contains no particle positions!", + ablastr::warn_manager::WarnPriority::high + ); + } } #ifdef WARPX_DIM_RZ diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 12aa46de058..99d6e0682ab 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -218,11 +218,13 @@ private: /** This function sets up the entries for storing the particle positions and global IDs * * @param[in] currSpecies Corresponding openPMD species + * @param[in] positionComponents user-selected components of the particle position * @param[in] np Number of particles * @param[in] isBTD Is this a back-transformed diagnostics output? */ void SetupPos ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, bool isBTD = false); @@ -231,12 +233,14 @@ private: * Sets the entries for storing particle position offset, constant records (charge, mass) and ED-PIC attributes. * * @param[in] currSpecies Corresponding openPMD species + * @param[in] positionComponents user-selected components of the particle position * @param[in] np Number of particles * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles */ void SetConstParticleRecordsEDPIC ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, amrex::ParticleReal charge, amrex::ParticleReal mass); @@ -284,8 +288,8 @@ private: * @param[in] name species name * @param[in] iteration timestep * @param[in] write_real_comp The real attribute ids, from WarpX - * @param[in] real_comp_names The real attribute names, from WarpX * @param[in] write_int_comp The int attribute ids, from WarpX + * @param[in] real_comp_names The real attribute names, from WarpX * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 965eb82b31e..45d5dc1cdee 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -213,32 +213,29 @@ namespace detail return make_pair(record_name, component_name); } - /** Return the component labels for particle positions + /** Return the user-selected components for particle positions + * + * @param[in] write_real_comp The real attribute ids, from WarpX + * @param[in] real_comp_names The real attribute names, from WarpX */ inline std::vector< std::string > - getParticlePositionComponentLabels (bool ignore_dims=false) + getParticlePositionComponentLabels ( + amrex::Vector const & write_real_comp, + amrex::Vector const & real_comp_names) { - using vs = std::vector< std::string >; - auto positionComponents = vs{"x", "y", "z"}; - if (!ignore_dims) { -#if defined(WARPX_DIM_1D_Z) - positionComponents = vs{"z"}; -#elif defined(WARPX_DIM_XZ) - positionComponents = vs{"x", "z"}; -#elif defined(WARPX_DIM_RZ) - // note: although we internally store particle positions - // for AMReX in r,z and a theta attribute, we - // actually need them for algorithms (e.g. push) - // and I/O in Cartesian. - // Other attributes like momentum are consequently - // stored in x,y,z internally. - positionComponents = vs{"x", "y", "z"}; -#elif defined(WARPX_DIM_3D) - positionComponents = vs{"x", "y", "z"}; -#else -# error Unknown WarpX dimensionality. -#endif + std::vector< std::string > positionComponents; + + int idx = 0; + for (auto const & comp : real_comp_names ) { + if (write_real_comp[idx]) { + if (comp == "position_x" || comp == "position_y" || comp == "position_z") { + std::string const last_letter{comp.back()}; + positionComponents.push_back(last_letter); + } + } + idx++; } + return positionComponents; } @@ -705,16 +702,18 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; } + auto const positionComponents = detail::getParticlePositionComponentLabels(write_real_comp, real_comp_names); + // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 if (doParticleSetup || is_resizing_flush) { - SetupPos(currSpecies, NewParticleVectorSize, isBTD); + SetupPos(currSpecies, positionComponents, NewParticleVectorSize, isBTD); SetupRealProperties(pc, currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, NewParticleVectorSize, isBTD); } if (is_last_flush_to_step) { - SetConstParticleRecordsEDPIC(currSpecies, NewParticleVectorSize, charge, mass); + SetConstParticleRecordsEDPIC(currSpecies, positionComponents, NewParticleVectorSize, charge, mass); } // open files from all processors, in case some will not contribute below @@ -978,6 +977,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, void WarpXOpenPMDPlot::SetupPos ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, bool const isBTD) { @@ -986,7 +986,6 @@ WarpXOpenPMDPlot::SetupPos ( auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); - auto const positionComponents = detail::getParticlePositionComponentLabels(); for( auto const& comp : positionComponents ) { currSpecies["position"][comp].resetDataset( realType ); } @@ -998,6 +997,7 @@ WarpXOpenPMDPlot::SetupPos ( void WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, amrex::ParticleReal const charge, amrex::ParticleReal const mass) @@ -1006,7 +1006,6 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles - auto const positionComponents = detail::getParticlePositionComponentLabels(true); for( auto const& comp : positionComponents ) { currSpecies["positionOffset"][comp].resetDataset( realType ); } @@ -1015,9 +1014,12 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( #if defined(WARPX_DIM_1D_Z) currSpecies["position"]["x"].resetDataset( realType ); currSpecies["position"]["y"].resetDataset( realType ); + currSpecies["positionOffset"]["x"].resetDataset( realType ); + currSpecies["positionOffset"]["y"].resetDataset( realType ); #endif #if defined(WARPX_DIM_XZ) currSpecies["position"]["y"].resetDataset( realType ); + currSpecies["positionOffset"]["y"].resetDataset( realType ); #endif // make constant @@ -1031,22 +1033,29 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( #if defined(WARPX_DIM_1D_Z) currSpecies["position"]["x"].makeConstant( 0._prt ); currSpecies["position"]["y"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["x"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["y"].makeConstant( 0._prt ); #endif #if defined(WARPX_DIM_XZ) currSpecies["position"]["y"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["y"].makeConstant( 0._prt ); #endif // meta data - currSpecies["position"].setUnitDimension( detail::getUnitDimension("position") ); - currSpecies["positionOffset"].setUnitDimension( detail::getUnitDimension("positionOffset") ); + if (!positionComponents.empty()) { + currSpecies["position"].setUnitDimension( detail::getUnitDimension("position") ); + currSpecies["positionOffset"].setUnitDimension( detail::getUnitDimension("positionOffset") ); + } currSpecies["charge"].setUnitDimension( detail::getUnitDimension("charge") ); currSpecies["mass"].setUnitDimension( detail::getUnitDimension("mass") ); // meta data for ED-PIC extension - currSpecies["position"].setAttribute( "macroWeighted", 0u ); - currSpecies["position"].setAttribute( "weightingPower", 0.0 ); - currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); - currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + if (!positionComponents.empty()) { + currSpecies["position"].setAttribute( "macroWeighted", 0u ); + currSpecies["position"].setAttribute( "weightingPower", 0.0 ); + currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); + currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + } currSpecies["id"].setAttribute( "macroWeighted", 0u ); currSpecies["id"].setAttribute( "weightingPower", 0.0 ); currSpecies["charge"].setAttribute( "macroWeighted", 0u ); From 02f38975c8104347804bd133df12118e014825b7 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Sun, 12 May 2024 12:18:44 -0700 Subject: [PATCH 107/121] Resetting charge buffers to zero. (#4916) * Resetting charge buffers to zero. * Cleaning up: added braces. * Added CI test ElectrostaticSphereLabFrame_MR_emass_10 (with large particle's mass). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Reverted ES sphere input script. * Docs: Fix PASC Paper Formatting * Highlights: Fix Formatting Issue Render issue in RST in PASC24 entry. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated benchmarks for ElectrostaticSphereLabFrame_MR_emass_10. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- .../analysis_electrostatic_sphere.py | 17 ++++++++++---- ...ectrostaticSphereLabFrame_MR_emass_10.json | 23 +++++++++++++++++++ Regression/WarpX-tests.ini | 17 ++++++++++++++ Source/FieldSolver/ElectrostaticSolver.cpp | 16 +++++++++++-- 4 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index 8e05bfa0c68..e63a3f02ba8 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -18,6 +18,7 @@ inverse t(r) can be solved for exactly. """ import os +import re import sys import numpy as np @@ -36,6 +37,15 @@ ds = yt.load( filename ) t_max = ds.current_time.item() # time of simulation +# Parse test name and check if particle_shape = 4 is used +emass_10 = True if re.search('emass_10', filename) else False + +if emass_10: + l2_tolerance = 0.096 + m_e = 10 +else: + l2_tolerance = 0.05 + m_e = 9.10938356e-31 #Electron mass in kg ndims = np.count_nonzero(ds.domain_dimensions > 1) if ndims == 2: @@ -60,7 +70,6 @@ # Constants eps_0 = 8.8541878128e-12 #Vacuum Permittivity in C/(V*m) -m_e = 9.10938356e-31 #Electron mass in kg q_e = -1.60217662e-19 #Electron charge in C pi = np.pi #Circular constant of the universe r_0 = 0.1 #Initial radius of sphere @@ -134,9 +143,9 @@ def calculate_error(E_axis, xmin, dx, nx): print("L2 error along y-axis = %s" %L2_error_y) print("L2 error along z-axis = %s" %L2_error_z) -assert L2_error_x < 0.05 -assert L2_error_y < 0.05 -assert L2_error_z < 0.05 +assert L2_error_x < l2_tolerance +assert L2_error_y < l2_tolerance +assert L2_error_z < l2_tolerance # Check conservation of energy def return_energies(iteration): diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json new file mode 100644 index 00000000000..024127a1bd2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json @@ -0,0 +1,23 @@ +{ + "lev=0": { + "Ex": 7.110909624093144, + "Ey": 7.110909624093145, + "Ez": 7.110909624093143, + "rho": 0.0 + }, + "lev=1": { + "Ex": 14.281015560380963, + "Ey": 14.281015560380965, + "Ez": 14.281015560380965, + "rho": 2.6092568008333786e-10 + }, + "electron": { + "particle_momentum_x": 1.80842228672388e-24, + "particle_momentum_y": 1.8084222867238806e-24, + "particle_momentum_z": 1.7598771525647628e-24, + "particle_position_x": 327.46875, + "particle_position_y": 327.46875, + "particle_position_z": 327.46875, + "particle_weight": 6212.501525878906 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index d31a042c807..16382d8f338 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -465,6 +465,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +[ElectrostaticSphereLabFrame_MR_emass_10] +buildDir = . +inputFile = Examples/Tests/electrostatic_sphere/inputs_3d +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 +dim = 3 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=3 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py + [ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_3d diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 5bb50ada278..a74beae3c91 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -195,7 +195,13 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } - + for (int lev = 0; lev <= max_level; lev++) { + if (lev > 0) { + if (charge_buf[lev]) { + charge_buf[lev]->setVal(0.); + } + } + } SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels // Get the particle beta vector @@ -239,7 +245,13 @@ WarpX::AddSpaceChargeFieldLabFrame () int const lev = 0; myfl->DepositCharge( lev, *rho_fp[lev] ); } - + for (int lev = 0; lev <= max_level; lev++) { + if (lev > 0) { + if (charge_buf[lev]) { + charge_buf[lev]->setVal(0.); + } + } + } SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels #ifndef WARPX_DIM_RZ for (int lev = 0; lev <= finestLevel(); lev++) { From d3069053d1eae9c44db5ae92380108eab1d4615d Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Sun, 12 May 2024 13:24:57 -0700 Subject: [PATCH 108/121] Update hip installation script for CI (#4927) It appears that "apt/debian ubuntu main" does not work anymore. We now use "apt/latest ${UBUNTU_CODENAME} main" instead. --- .github/workflows/dependencies/hip.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2225e670bb0..ae897786acd 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -12,12 +12,24 @@ set -eu -o pipefail # failed files the given number of times. echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries -# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu -wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key \ - | sudo apt-key add - -echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \ - | sudo tee /etc/apt/sources.list.d/rocm.list +# Ref.: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/ubuntu.html + +# Make the directory if it doesn't exist yet. +# This location is recommended by the distribution maintainers. +sudo mkdir --parents --mode=0755 /etc/apt/keyrings + +# Download the key, convert the signing-key to a full +# keyring required by apt and store in the keyring directory +wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ + gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null +curl -O https://repo.radeon.com/rocm/rocm.gpg.key +sudo apt-key add rocm.gpg.key + +source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ... + +echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \ + | sudo tee /etc/apt/sources.list.d/rocm.list echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ | sudo tee -a /etc/profile.d/rocm.sh From 4270ca8521cd9a48d0d2fddcb174153c4d555b83 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 12 May 2024 21:58:45 -0700 Subject: [PATCH 109/121] `ElectrostaticSphereLabFrame_MR_emass_10`: +Positions (#4930) --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 16382d8f338..0f3957a8cce 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -468,7 +468,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sph [ElectrostaticSphereLabFrame_MR_emass_10] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 From 2a3fbdacbfb7edfd279c4c92543597bf0b475dd3 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 13 May 2024 13:15:14 +0200 Subject: [PATCH 110/121] ablastr::particles::compute_weights : implement 1D and use template parameter to specify if field is nodal (#4846) * use template parameter to specify if field is nodal in compute_weights and implement function also for 1D case * fix bug * remove unused variables * use amrex::IndexType::NODE as template parameter * Add Missing Include * Doc: Remove `(default)` Co-authored-by: Axel Huebl --- Source/Diagnostics/ParticleIO.cpp | 3 +- Source/EmbeddedBoundary/ParticleScraper.H | 7 +-- Source/Particles/ParticleBoundaryBuffer.cpp | 10 ++-- Source/ablastr/particles/NodalFieldGather.H | 58 +++++++++++++-------- 4 files changed, 48 insertions(+), 30 deletions(-) diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index c4164e383b1..f1f4d426a4a 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -283,7 +283,8 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, getPosition(ip, xp, yp, zp); int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); phi_particle_arr[ip] = phi_value; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index ea4d0bc5c0e..1e915b39381 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -183,15 +183,16 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { int ic, jc, kc; // Cell-centered indices - [[maybe_unused]] int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, ic, jc, kc, Wc); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); DistanceToEB::normalize(normal); amrex::RealVect pos; diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index c8c683f0abf..345a0bfa592 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -98,7 +98,8 @@ struct FindEmbeddedBoundaryIntersection { amrex::Real W[AMREX_SPACEDIM][2]; amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); return phi_value; } ); @@ -115,11 +116,12 @@ struct FindEmbeddedBoundaryIntersection { // record the components of the normal on the destination int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); int ic, jc, kc; // Cell-centered indices - int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weight - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc, nodal=0); // nodal=0 to calculate the weights with respect to the cell-centered nodes + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phiarr, dxi); DistanceToEB::normalize(normal); diff --git a/Source/ablastr/particles/NodalFieldGather.H b/Source/ablastr/particles/NodalFieldGather.H index 5a10c73ae20..f324264db7f 100644 --- a/Source/ablastr/particles/NodalFieldGather.H +++ b/Source/ablastr/particles/NodalFieldGather.H @@ -12,16 +12,19 @@ #include #include #include +#include #include #include + namespace ablastr::particles { + /** * \brief Compute weight of each surrounding node (or cell-centered nodes) in interpolating a nodal ((or a cell-centered node) field - * to the given coordinates. If nodal=1, then the calculations will be done with respect to the nodes (default). If nodal=0, then the calculations will be done with respect to the cell-centered nodal) - * - * This currently only does linear order. + * to the given coordinates. If template parameter IdxType is amrex::IndexType::CellIndex::NODE, then the calculations will be done with respect to the nodes. + * If template parameter IdxType is amrex::IndexType::CellIndex::CELL, then the calculations will be done with respect to the cell-centered nodal. + * This currently only does linear order. * * \param xp,yp,zp Particle position coordinates * \param plo Index lower bounds of domain. @@ -30,24 +33,30 @@ namespace ablastr::particles * \param W 2D array of weights to store each neighbouring node (or cell-centered node) * \param nodal Int that tells if the weights are calculated in respect to the nodes (nodal=1) of the cell-centered nodes (nodal=0) */ +template AMREX_GPU_HOST_DEVICE AMREX_INLINE void compute_weights (const amrex::ParticleReal xp, const amrex::ParticleReal yp, const amrex::ParticleReal zp, amrex::GpuArray const& plo, amrex::GpuArray const& dxi, - int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2], int nodal=1) noexcept + int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2]) noexcept { using namespace amrex::literals; -#if !((nodal==0)||(nodal==1)) - ABLASTR_ABORT_WITH_MESSAGE("Error: 'nodal' has to be equal to 0 or 1"); -#endif + constexpr auto half_if_cell_centered = [](){ + if constexpr (IdxType == amrex::IndexType::CellIndex::NODE){ + return 0.0_rt; + } + else{ + return 0.5_rt; + } + }(); #if (defined WARPX_DIM_3D) - const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; - const amrex::Real y = (yp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; - const amrex::Real z = (zp - plo[2]) * dxi[2] + static_cast(nodal-1)*0.5_rt; + const amrex::Real x = (xp - plo[0]) * dxi[0] - half_if_cell_centered; + const amrex::Real y = (yp - plo[1]) * dxi[1] - half_if_cell_centered; + const amrex::Real z = (zp - plo[2]) * dxi[2] - half_if_cell_centered; i = static_cast(amrex::Math::floor(x)); j = static_cast(amrex::Math::floor(y)); @@ -64,17 +73,17 @@ void compute_weights (const amrex::ParticleReal xp, #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) # if (defined WARPX_DIM_XZ) - const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real x = (xp - plo[0]) * dxi[0] - half_if_cell_centered; amrex::ignore_unused(yp); i = static_cast(amrex::Math::floor(x)); W[0][1] = x - i; # elif (defined WARPX_DIM_RZ) - const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] - half_if_cell_centered; i = static_cast(amrex::Math::floor(r)); W[0][1] = r - i; # endif - const amrex::Real z = (zp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; + const amrex::Real z = (zp - plo[1]) * dxi[1] - half_if_cell_centered; j = static_cast(amrex::Math::floor(z)); W[1][1] = z - j; @@ -83,8 +92,15 @@ void compute_weights (const amrex::ParticleReal xp, k = 0; #else - amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W, nodal); - ABLASTR_ABORT_WITH_MESSAGE("Error: compute_weights not yet implemented in 1D"); + const amrex::Real z = (zp - plo[0]) * dxi[0] - half_if_cell_centered; + amrex::ignore_unused(xp, yp); + i = static_cast(amrex::Math::floor(z)); + + W[0][1] = z - i; + W[0][0] = 1.0_rt - W[0][1]; + + j = 0; + k = 0; #endif } @@ -100,8 +116,8 @@ amrex::Real interp_field_nodal (int i, int j, int k, const amrex::Real W[AMREX_SPACEDIM][2], amrex::Array4 const& scalar_field) noexcept { -#if (defined WARPX_DIM_3D) amrex::Real value = 0; +#if (defined WARPX_DIM_3D) value += scalar_field(i, j , k ) * W[0][0] * W[1][0] * W[2][0]; value += scalar_field(i+1, j , k ) * W[0][1] * W[1][0] * W[2][0]; value += scalar_field(i, j+1, k ) * W[0][0] * W[1][1] * W[2][0]; @@ -111,15 +127,13 @@ amrex::Real interp_field_nodal (int i, int j, int k, value += scalar_field(i , j+1, k+1) * W[0][0] * W[1][1] * W[2][1]; value += scalar_field(i+1, j+1, k+1) * W[0][1] * W[1][1] * W[2][1]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Real value = 0; value += scalar_field(i, j , k) * W[0][0] * W[1][0]; value += scalar_field(i+1, j , k) * W[0][1] * W[1][0]; value += scalar_field(i, j+1, k) * W[0][0] * W[1][1]; value += scalar_field(i+1, j+1, k) * W[0][1] * W[1][1]; #else - const amrex::Real value = 0; - amrex::ignore_unused(i, j, k, W, scalar_field); - ABLASTR_ABORT_WITH_MESSAGE("Error: interp_field not yet implemented in 1D"); + value += scalar_field(i, j , k) * W[0][0]; + value += scalar_field(i+1, j , k) * W[0][1]; #endif return value; } @@ -144,7 +158,7 @@ amrex::Real doGatherScalarFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); return interp_field_nodal(ii, jj, kk, W, scalar_field); } @@ -172,7 +186,7 @@ doGatherVectorFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); amrex::GpuArray const field_interp = { interp_field_nodal(ii, jj, kk, W, vector_field_x), From ee51424e1a875332967a9302df249cb521448fa1 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 13 May 2024 11:24:19 -0700 Subject: [PATCH 111/121] Remove redundant setVal(0) in ParticleReductionFunctor (#4926) ParticleReductionFunctor uses the routine ParticleToMesh, which by default will zero out the MultiFab passed in. This PR removes the unneeded explicit calls to setVal(0.). --- .../ComputeDiagFunctors/ParticleReductionFunctor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 15473890233..34fbbee5c7c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -52,8 +52,6 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, constexpr int ng = 1; // Temporary cell-centered, multi-component MultiFab for storing particles per cell. amrex::MultiFab red_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); - // Set value to 0, and increment the value in each cell with ppc. - red_mf.setVal(0._rt); auto& pc = warpx.GetPartContainer().GetParticleContainer(m_ispec); // Copy over member variables so they can be captured in the lambda auto map_fn = m_map_fn; @@ -99,7 +97,6 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, }); if (m_do_average) { amrex::MultiFab ppc_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); - ppc_mf.setVal(0._rt); // Add the weight for each particle -- total number of particles of this species ParticleToMesh(pc, ppc_mf, m_lev, [=] AMREX_GPU_DEVICE (const WarpXParticleContainer::SuperParticleType& p, From c70a6c59f00bb8eb34b4d8d53712fb84d1ebb506 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 13 May 2024 15:48:07 -0700 Subject: [PATCH 112/121] Code clean-up for binary collisions (#4921) * code clean-up for binary collisions * fix variable shadowing * pass particle tiles by reference --- .../BinaryCollision/BinaryCollision.H | 6 +-- .../BinaryCollision/BinaryCollisionUtils.H | 28 ++++++++++++ .../DSMC/SplitAndScatterFunc.H | 45 ++++--------------- .../BinaryCollision/ParticleCreationFunc.H | 45 +++++-------------- 4 files changed, 48 insertions(+), 76 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index bd6ec84dd69..b3d39a9c581 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -75,7 +75,6 @@ class BinaryCollision final using ParticleTileType = WarpXParticleContainer::ParticleTileType; using ParticleTileDataType = ParticleTileType::ParticleTileDataType; using ParticleBins = amrex::DenseBins; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; public: @@ -298,7 +297,6 @@ public: auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); #endif - /* The following calculations are only required when creating product particles */ @@ -431,7 +429,7 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, + ptile_1, ptile_1, product_species_vector, tile_products_data, m1, m1, @@ -648,7 +646,7 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, + ptile_1, ptile_2, product_species_vector, tile_products_data, m1, m2, diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 091bd41955f..08ddcd27174 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -126,6 +126,34 @@ namespace BinaryCollisionUtils{ // is calculated here. lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); } + + /** + * \brief Subtract given weight from particle and set its ID to invalid + * if the weight reaches zero. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void remove_weight_from_colliding_particle ( + amrex::ParticleReal& weight, uint64_t& idcpu, + const amrex::ParticleReal reaction_weight ) + { + // Remove weight from given particle + amrex::Gpu::Atomic::AddNoRet(&weight, -reaction_weight); + + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to invalid + if (weight <= std::numeric_limits::min()) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + } + } } #endif // WARPX_BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index 86a2c70374e..a0c6ca8c8d4 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -27,8 +27,7 @@ class SplitAndScatterFunc using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; using ParticleBins = amrex::DenseBins; using index_type = typename ParticleBins::index_type; - using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: /** @@ -53,8 +52,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& n_total_pairs, - // Tile& ptile1, Tile& ptile2, - const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + ParticleTileType& ptile1, ParticleTileType& ptile2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, const amrex::ParticleReal m1, const amrex::ParticleReal m2, @@ -93,9 +91,8 @@ public: tile_products[i]->resize(products_np[i] + num_added); } - // this works for DSMC since the colliding particles are also products - const auto soa_1 = tile_products[0]->getParticleTileData(); - const auto soa_2 = tile_products[1]->getParticleTileData(); + const auto soa_1 = ptile1.getParticleTileData(); + const auto soa_2 = ptile2.getParticleTileData(); amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; @@ -155,36 +152,10 @@ public: soa_products_data[1].m_rdata[PIdx::w][product2_index] = p_pair_reaction_weight[i]; // Remove p_pair_reaction_weight[i] from the colliding particles' weights - amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], - -p_pair_reaction_weight[i]); - amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], - -p_pair_reaction_weight[i]); - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; -#else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); -#endif - }; - - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to invalid - if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); - } - if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); - } + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); // Set the child particle properties appropriately auto& ux1 = soa_products_data[0].m_rdata[PIdx::ux][product1_index]; diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index bf31076d975..3cb7197b93a 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -67,8 +67,7 @@ public: * function specific to the considered binary collision. * * @param[in] n_total_pairs how many binary collisions have been performed in this tile - * @param[in, out] soa_1 struct of array data of the first colliding particle species - * @param[in, out] soa_2 struct of array data of the second colliding particle species + * @param[in, out] ptile1,ptile2 the particle tiles of the two colliding species * @param[out] tile_products array containing tile data of the product particles. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species @@ -94,7 +93,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& n_total_pairs, - const SoaData_type& soa_1, const SoaData_type& soa_2, + ParticleTileType& ptile1, ParticleTileType& ptile2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, @@ -129,6 +128,9 @@ public: tile_products[i]->resize(products_np[i] + num_added); } + const auto soa_1 = ptile1.getParticleTileData(); + const auto soa_2 = ptile2.getParticleTileData(); + amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; @@ -196,37 +198,10 @@ public: } // Remove p_pair_reaction_weight[i] from the colliding particles' weights - amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], - -p_pair_reaction_weight[i]); - amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], - -p_pair_reaction_weight[i]); - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; -#else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); -#endif - }; - - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to invalid - if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); - - } - if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); - } + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); // Initialize the product particles' momentum, using a function depending on the // specific collision type @@ -323,7 +298,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& /*n_total_pairs*/, - const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + ParticleTileType& /*ptile1*/, ParticleTileType& /*ptile2*/, amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, From 0a1284c18fcdc97a683fcec4962253a370fdc27e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 13 May 2024 15:48:39 -0700 Subject: [PATCH 113/121] Release 24.05 (#4928) * AMReX: 24.05 * pyAMReX: 24.05 * WarpX: 24.05 * `Python_restart_eb`: Reset Checksum --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- .../benchmarks_json/Python_restart_eb.json | 14 +++++++------- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f73c39b6fc3..eaff5543c78 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - + cd ../amrex && git checkout --detach 24.05 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 92f235d91ec..31b410fc55e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.04) +project(WarpX VERSION 24.05) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 7299b865746..471f6f2b6a3 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.04' +version = u'24.05' # The full version, including alpha/beta/rc tags. -release = u'24.04' +release = u'24.05' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index d1b8ddd27e5..fbf3330ada0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.04', + version = '24.05', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/Checksum/benchmarks_json/Python_restart_eb.json b/Regression/Checksum/benchmarks_json/Python_restart_eb.json index 25087c5757d..ad0d2cee5a3 100644 --- a/Regression/Checksum/benchmarks_json/Python_restart_eb.json +++ b/Regression/Checksum/benchmarks_json/Python_restart_eb.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 148618.63186220315, - "By": 148618.63186220315, - "Bz": 3385.851454453729, - "Ex": 55362729623335.9, - "Ey": 55362729623335.89, - "Ez": 68396725892689.86 + "Bx": 148673.005859208, + "By": 148673.00585920806, + "Bz": 3371.758117878558, + "Ex": 55378581103426.71, + "Ey": 55378581103426.72, + "Ez": 68412803445328.25 } -} \ No newline at end of file +} diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 94b334d02da..c188bc5ea45 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = d857968968ae2cdf18f39ee50df96529e326ff39 +branch = 24.05 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 0f3957a8cce..085e9e0220e 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = d857968968ae2cdf18f39ee50df96529e326ff39 +branch = 24.05 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index cbde4bca135..c2e1828b737 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.04 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.05 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "d857968968ae2cdf18f39ee50df96529e326ff39" +set(WarpX_amrex_branch "24.05" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index fcf7c91bce2..a42ffa6ba41 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.04 CONFIG REQUIRED) + find_package(pyAMReX 24.05 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "f370d38ab410dbf1b3a90683c97291841ebeb3f5" +set(WarpX_pyamrex_branch "24.05" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 75e56e6964f..001fe1094fe 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - +cd amrex && git checkout --detach 24.05 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 70b333fc6b9..164f4f1395f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.04', + version = '24.05', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From b8c8d394801c117827a126b66943aa9b832d5d08 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 14 May 2024 05:47:33 -0700 Subject: [PATCH 114/121] LICENSE.txt: Wrap to 80 Characters (#4929) Use modern formatting as we have in newer codes to wrap the `LICENSE.txt` file for readability to 80 characters per line. No content change. --- LICENSE.txt | 60 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 9d9b939f342..2965985ebb1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,16 +1,44 @@ -WarpX Copyright (c) 2018, The Regents of the University of California, through Lawrence Berkeley National Laboratory, and Lawrence Livermore National Security, LLC, for the operation of Lawrence Livermore National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. - - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - -(1) Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -(2) Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -(3) Neither the name of the University of California, Lawrence Berkeley National Laboratory, Lawrence Livermore National Security, LLC, Lawrence Livermore National Laboratory, U.S. Dept. of Energy, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -You are under no obligation whatsoever to provide any bug fixes, patches, or upgrades to the features, functionality or performance of the source code ("Enhancements") to anyone; however, if you choose to make your Enhancements available either publicly, or directly to Lawrence Berkeley National Laboratory, without imposing a separate written license agreement for such Enhancements, then you hereby grant the following license: a non-exclusive, royalty-free perpetual license to install, use, modify, prepare derivative works, incorporate into other computer software, distribute, and sublicense such enhancements or derivative works thereof, in binary and source code form. +WarpX Copyright (c) 2018, The Regents of the University of California, +through Lawrence Berkeley National Laboratory, and Lawrence Livermore National +Security, LLC, for the operation of Lawrence Livermore National Laboratory +(subject to receipt of any required approvals from the U.S. Dept. of Energy). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +(1) Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +(2) Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +(3) Neither the name of the University of California, Lawrence Berkeley +National Laboratory, U.S. Dept. of Energy nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +You are under no obligation whatsoever to provide any bug fixes, patches, +or upgrades to the features, functionality or performance of the source +code ("Enhancements") to anyone; however, if you choose to make your +Enhancements available either publicly, or directly to Lawrence Berkeley +National Laboratory, without imposing a separate written license agreement +for such Enhancements, then you hereby grant the following license: a +non-exclusive, royalty-free perpetual license to install, use, modify, +prepare derivative works, incorporate into other computer software, +distribute, and sublicense such enhancements or derivative works thereof, +in binary and source code form. From 42c3e2802416bf4e97d072be67bfee284c0531b2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 14 May 2024 08:56:14 -0700 Subject: [PATCH 115/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4931) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index eaff5543c78..b54afde491b 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.05 && cd - + cd ../amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index c188bc5ea45..5e95dd04eb8 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.05 +branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 085e9e0220e..fcd35bbbe7d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.05 +branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index c2e1828b737..146b6003729 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.05" +set(WarpX_amrex_branch "8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a42ffa6ba41..f85af8727d4 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.05" +set(WarpX_pyamrex_branch "74d93b53d60b25200e36f345f7b0ac5d611512b9" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 001fe1094fe..470a497979b 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.05 && cd - +cd amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From ff2516f4cefddfb01c2e9935dc059f9e07d99dde Mon Sep 17 00:00:00 2001 From: Daniel <92740330+PhysicsDan@users.noreply.github.com> Date: Wed, 15 May 2024 20:54:21 +0100 Subject: [PATCH 116/121] Updated proton-boron cross section (#4377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * pB Fusion: Updated reaction cross sections * Updated the analytical model for pB fusion reaction cross sections to match that described in A. Tentori and F. Belloni Nucl. Fusion 63, 086001 (2023). * Updated python analysis test scripts to match new cross section function * Fixed reference in analysis_proton_boron_fusion.py * Bug: Fixed typo in analysis script Typo in constant C1 for low temp nonresonant pB reactivity * Updated checksums for pB fusion Updated the checksums for pB fusion 2D and 3D tests using the pB cross section data from A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023) * Update Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py Fixed comment to note the energy range covered by the XS fit in A. Tentori & F. Belloni Nucl. Fusion 63 086001 (2023). Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> * Fix pB crosssection for E>9.76 MeV * Added the power law decribed in Buck et al. Nuclear Physics A, 398(2), 189-202 (1983) for E>9.76 MeV * Note: cross section starting value altered from paper to ensure continuity with E<=9.76 MeV * Update Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> * update checksum values for pB11 fusion tests --------- Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: roelof-groenewald --- .../analysis_proton_boron_fusion.py | 88 +++---- .../Proton_Boron_Fusion_2D.json | 158 ++++++------ .../Proton_Boron_Fusion_3D.json | 226 +++++++++--------- .../ProtonBoronFusionCrossSection.H | 76 +++--- 4 files changed, 274 insertions(+), 274 deletions(-) diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index eec2ba4fffb..543eb62484a 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -40,8 +40,8 @@ ## the initial number of protons and borons. ## ## The third test corresponds to a Maxwellian plasma with a 44 keV temperature. The alpha yield is -## directly compared to the analytical fits of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 -## (2000) for a thermal plasma. +## directly compared to the analytical fits of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 +## (2023) for a thermal plasma. ## ## The fourth test corresponds to a plasma with an extremely small boron density, so that all boron ## macroparticles should have disappeared by the end of the simulation, which we verify. @@ -259,10 +259,10 @@ def check_isotropy(data, relative_tolerance): def astrophysical_factor_lowE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the low energy fit in the range E < 400 keV - ## described in equation (2) of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000) + ## described in equation (3) of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023) C0 = 197. - C1 = 0.24 - C2 = 2.31e-4 + C1 = 0.269 + C2 = 2.54e-4 AL = 1.82e4 EL = 148. dEL = 2.35 @@ -271,12 +271,12 @@ def astrophysical_factor_lowE(E): def astrophysical_factor_midE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the mid energy fit in the range - ## 400 keV < E < 642 keV described in equation (3) of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) - D0 = 330. - D1 = 66.1 - D2 = -20.3 - D5 = -1.58 + ## 400 keV < E < 668 keV described in equation (4) of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023) + D0 = 346. + D1 = 150. + D2 = -59.9 + D5 = -0.460 E_400 = 400. E_100 = 100. E_norm = (E - E_400)/E_100 @@ -285,29 +285,29 @@ def astrophysical_factor_midE(E): def astrophysical_factor_highE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the high energy fit in the range - ## 642 keV < E < 3500 keV described in equation (4) of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) - A0 = 2.57e6 - A1 = 5.67e5 - A2 = 1.34e5 - A3 = 5.68e5 - E0 = 581.3 - E1 = 1083. - E2 = 2405. - E3 = 3344. - dE0 = 85.7 - dE1 = 234. - dE2 = 138. - dE3 = 309. - B = 4.38 + ## 668 keV < E < 9760 keV described in equation (5) of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023) + A0 = 1.98e6 + A1 = 3.89e6 + A2 = 1.36e6 + A3 = 3.71e6 + E0 = 640.9 + E1 = 1211. + E2 = 2340. + E3 = 3294. + dE0 = 85.5 + dE1 = 414. + dE2 = 221. + dE3 = 351. + B = 0.381 return A0/((E-E0)**2 + dE0**2) + A1/((E-E1)**2 + dE1**2) + \ A2/((E-E2)**2 + dE2**2) + A3/((E-E3)**2 + dE3**2) + B def astrophysical_factor(E): ## E is in keV - ## Returns astrophysical factor in MeV b using the fits described in W.M. Nevins - ## and R. Swain, Nuclear Fusion, 40, 865 (2000) - conditions = [E <= 400, E <= 642, E > 642] + ## Returns astrophysical factor in MeV b using the fits described in A. Tentori + ## and F. Belloni, Nuclear Fusion, 63, 086001 (2023) + conditions = [E <= 400, E <= 668, E > 668] choices = [astrophysical_factor_lowE(E), astrophysical_factor_midE(E), astrophysical_factor_highE(E)] @@ -316,20 +316,20 @@ def astrophysical_factor(E): def pb_cross_section_buck_fit(E): ## E is in MeV ## Returns cross section in b using a power law fit of the data presented in Buck et al., - ## Nuclear Physics A, 398(2), 189-202 (1983) in the range E > 3.5 MeV. - E_start_fit = 3.5 + ## Nuclear Physics A, 398(2), 189-202 (1983) in the range E > 9.76 MeV. + E_start_fit = 9.76 ## Cross section at E = E_start_fit = 3.5 MeV - cross_section_start_fit = 0.2168440845211521 + cross_section_start_fit = 0.01277998 slope_fit = -2.661840717596765 return cross_section_start_fit*(E/E_start_fit)**slope_fit def pb_cross_section(E): ## E is in keV - ## Returns cross section in b using the fits described in W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) for E < 3.5 MeV and a power law fit of the data presented in - ## Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) for E > 3.5 MeV. + ## Returns cross section in b using the fits described in A. Tentori and F. Belloni, + ## Nucl. Fusion, 63, 086001 (2023) for E < 9.76 MeV otherwise returns a power law fit + ## of the data in Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) E_MeV = E/1.e3 - conditions = [E <= 3500, E > 3500] + conditions = [E <= 9760, E > 9760] choices = [astrophysical_factor(E)/E_MeV * np.exp(-np.sqrt(E_Gamow_MeV / E_MeV)), pb_cross_section_buck_fit(E_MeV)] return np.select(conditions, choices) @@ -598,13 +598,13 @@ def check_xy_isotropy(data): def sigmav_thermal_fit_lowE_nonresonant(T): ## Temperature T is in keV ## Returns the nonresonant average of cross section multiplied by relative velocity in m^3/s, - ## in the range T <= 70 keV, as described by equation 9 of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000). + ## in the range T <= 70 keV, as described by equations 10-14 of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023). E0 = (E_Gamow_keV/4.)**(1./3.) * T**(2./3.) DE0 = 4.*np.sqrt(T*E0/3.) C0 = 197.*1.e3 - C1 = 0.24*1.e3 - C2 = 2.31e-4*1.e3 + C1 = 0.269*1.e3 + C2 = 2.54e-4*1.e3 tau = 3.*E0/T Seff = C0*(1.+5./(12.*tau)) + C1*(E0+35./36.*T) + C2*(E0**2 + 89./36.*E0*T) ## nonresonant sigma times vrel, in barn meter per second @@ -615,21 +615,21 @@ def sigmav_thermal_fit_lowE_nonresonant(T): def sigmav_thermal_fit_lowE_resonant(T): ## Temperature T is in keV ## Returns the resonant average of cross section multiplied by relative velocity in m^3/s, - ## in the range T <= 70 keV, as described by equation 11 of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000). + ## in the range T <= 70 keV, as described by equation 15 of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023). return 5.41e-21 * np.exp(-148./T) / T**(3./2.) def sigmav_thermal_fit_lowE(T): ## Temperature T is in keV ## Returns the average of cross section multiplied by relative velocity in m^3/s, using the - ## fits described in section 3.1 of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). + ## fits described in section 2.2 of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fits are valid for T <= 70 keV. return sigmav_thermal_fit_lowE_nonresonant(T) + sigmav_thermal_fit_lowE_resonant(T) def expected_alpha_thermal(T, proton_density, boron_density, dV, dt): ## Computes the expected number of produced alpha particles when the protons and borons follow ## a Maxwellian distribution with a temperature T, in keV. This uses the thermal fits described - ## in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). + ## in A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fit used here is only valid in the range T <= 70 keV. assert((T >=0) and (T<=70)) diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index c6bd572ab68..cdee4b078b0 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -2,29 +2,29 @@ "lev=0": { "rho": 0.0 }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789535533e+28 + "alpha2": { + "particle_momentum_x": 4.057984510682467e-15, + "particle_momentum_y": 4.104188139725855e-15, + "particle_momentum_z": 4.17858000090827e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 5.078061191951185e+18 }, - "alpha4": { - "particle_momentum_x": 2.3381598715816888e-14, - "particle_momentum_y": 2.3435337108040707e-14, - "particle_momentum_z": 2.3554073734365495e-14, - "particle_position_x": 2457367.4582781526, - "particle_position_y": 4915112.044373058, - "particle_weight": 384.0000000000002 + "alpha3": { + "particle_momentum_x": 5.017656304003558e-16, + "particle_momentum_y": 4.935595075276182e-16, + "particle_momentum_z": 4.867133212376827e-16, + "particle_position_x": 52678.192400911765, + "particle_position_y": 105483.59950020742, + "particle_weight": 1.5413633830148085e+27 }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399508749004609e+30 + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, + "particle_weight": 128.00000000000261 }, "boron2": { "particle_momentum_x": 0.0, @@ -32,39 +32,7 @@ "particle_momentum_z": 0.0, "particle_position_x": 409798.015821768, "particle_position_y": 819270.9858143466, - "particle_weight": 1.279999999903553e+28 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 - }, - "alpha2": { - "particle_momentum_x": 4.057984510682467e-15, - "particle_momentum_y": 4.104188139725855e-15, - "particle_momentum_z": 4.17858000090827e-15, - "particle_position_x": 408793.7905852193, - "particle_position_y": 861780.8020495367, - "particle_weight": 2.893405947366684e+18 - }, - "proton3": { - "particle_momentum_x": 1.6847528610914633e-15, - "particle_momentum_y": 1.6827441855152735e-15, - "particle_momentum_z": 1.6802807054711977e-15, - "particle_position_x": 2457284.001605452, - "particle_position_y": 4914327.937654163, - "particle_weight": 1.2795087490046132e+30 - }, - "alpha3": { - "particle_momentum_x": 4.969030674549252e-16, - "particle_momentum_y": 4.889174794354074e-16, - "particle_momentum_z": 4.826711561221412e-16, - "particle_position_x": 52379.50835085382, - "particle_position_y": 104954.74986555413, - "particle_weight": 1.4737529861620076e+27 + "particle_weight": 1.2799999998307316e+28 }, "boron5": { "particle_momentum_x": 0.0, @@ -74,21 +42,21 @@ "particle_position_y": 819118.5558814355, "particle_weight": 127.99999999999999 }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "alpha1": { + "particle_momentum_x": 4.691909092431811e-15, + "particle_momentum_y": 4.6836958163275755e-15, + "particle_momentum_z": 4.657203546376977e-15, + "particle_position_x": 463465.27131109464, + "particle_position_y": 978207.6061359186, + "particle_weight": 4.977661460251435e-28 }, - "boron1": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512788807322e+28 }, "proton1": { "particle_momentum_x": 0.0, @@ -98,20 +66,52 @@ "particle_position_y": 81919772.69310114, "particle_weight": 128.00000000000261 }, - "alpha1": { - "particle_momentum_x": 4.691909092431811e-15, - "particle_momentum_y": 4.6836958163275755e-15, - "particle_momentum_z": 4.657203546376977e-15, - "particle_position_x": 463465.27131109464, - "particle_position_y": 978207.6061359186, - "particle_weight": 2.964832324596836e-28 - }, "alpha5": { - "particle_momentum_x": 2.3343469003376247e-14, - "particle_momentum_y": 2.3450435277836452e-14, - "particle_momentum_z": 2.357999284036709e-14, + "particle_momentum_x": 2.3343485859070736e-14, + "particle_momentum_y": 2.3451128753701046e-14, + "particle_momentum_z": 2.3579462789062662e-14, "particle_position_x": 2457556.8571638423, - "particle_position_y": 4914659.635379323, + "particle_position_y": 4914659.635379322, "particle_weight": 3.839999999999998e-19 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399486212205656e+30 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 + }, + "proton3": { + "particle_momentum_x": 1.6847290893972251e-15, + "particle_momentum_y": 1.6827074502304075e-15, + "particle_momentum_z": 1.6802489646490975e-15, + "particle_position_x": 2457270.6999197667, + "particle_position_y": 4914315.665267942, + "particle_weight": 1.279486212205663e+30 + }, + "alpha4": { + "particle_momentum_x": 2.338084461204216e-14, + "particle_momentum_y": 2.3436156778849828e-14, + "particle_momentum_z": 2.35535708386288e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 4915112.044373056, + "particle_weight": 384.0000000000002 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index 9a078d754c3..ec7e047c537 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,131 +1,131 @@ { "lev=0": { - "rho": 0.0 + "rho": 0.0 }, - "alpha2": { - "particle_momentum_x": 4.1179548991012315e-15, - "particle_momentum_y": 4.110026665992801e-15, - "particle_momentum_z": 4.169802553223462e-15, - "particle_position_x": 408575.75269073684, - "particle_position_y": 413407.5155277014, - "particle_position_z": 863983.4313441743, - "particle_weight": 1.901545867395127e+19 + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 }, - "alpha4": { - "particle_momentum_x": 2.3389474647479255e-14, - "particle_momentum_y": 2.3425362750821152e-14, - "particle_momentum_z": 2.3515097401900785e-14, - "particle_position_x": 2457367.458278154, - "particle_position_y": 2457512.044373057, - "particle_position_z": 4914475.776513075, - "particle_weight": 3072.000000000002 + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "alpha3": { + "particle_momentum_x": 4.764404554793872e-16, + "particle_momentum_y": 4.655900875811434e-16, + "particle_momentum_z": 4.578927372510084e-16, + "particle_position_x": 50987.442011759704, + "particle_position_y": 48999.674675246955, + "particle_position_z": 101142.57224226737, + "particle_weight": 1.0633705344227063e+28 + }, + "alpha1": { + "particle_momentum_x": 4.665933695243743e-15, + "particle_momentum_y": 4.603805875733438e-15, + "particle_momentum_z": 4.706765986105302e-15, + "particle_position_x": 461871.79172011977, + "particle_position_y": 461162.2166206925, + "particle_position_z": 969262.7809050508, + "particle_weight": 3.2387855108185994e-27 + }, + "alpha5": { + "particle_momentum_x": 2.3388206254864998e-14, + "particle_momentum_y": 2.334372885765467e-14, + "particle_momentum_z": 2.363588638941874e-14, + "particle_position_x": 2457556.857163843, + "particle_position_y": 2457059.6353793247, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 }, "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 }, "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3745333755307162e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240721708e+29 - }, - "proton3": { - "particle_momentum_x": 1.6847690263373399e-15, - "particle_momentum_y": 1.682851264092124e-15, - "particle_momentum_z": 1.6803794796027466e-15, - "particle_position_x": 2457363.4111393224, - "particle_position_y": 2457049.324917686, - "particle_position_z": 4914549.051577593, - "particle_weight": 1.0236635064099502e+31 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.022781024024315e+29 }, "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 }, - "alpha5": { - "particle_momentum_x": 2.3387550695963677e-14, - "particle_momentum_y": 2.33445389960837e-14, - "particle_momentum_z": 2.3635948557453833e-14, - "particle_position_x": 2457556.8571638414, - "particle_position_y": 2457059.6353793237, - "particle_position_z": 4915847.043341329, - "particle_weight": 3.0719999999999984e-18 + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.1196455431551905e+31 }, - "alpha3": { - "particle_momentum_x": 4.72203056468584e-16, - "particle_momentum_y": 4.622060921861793e-16, - "particle_momentum_z": 4.5443204949049515e-16, - "particle_position_x": 50603.134609459696, - "particle_position_y": 48681.319634796026, - "particle_position_z": 100449.24933492322, - "particle_weight": 1.0094807701511393e+28 + "alpha4": { + "particle_momentum_x": 2.33898275612641e-14, + "particle_momentum_y": 2.3423797451957437e-14, + "particle_momentum_z": 2.3516107929259732e-14, + "particle_position_x": 2457367.458278153, + "particle_position_y": 2457512.0443730573, + "particle_position_z": 4914475.7765130745, + "particle_weight": 3072.000000000002 }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 + "proton3": { + "particle_momentum_x": 1.6847282386883186e-15, + "particle_momentum_y": 1.6828065767793222e-15, + "particle_momentum_z": 1.6803456707569493e-15, + "particle_position_x": 2457343.371083716, + "particle_position_y": 2457033.3891170574, + "particle_position_z": 4914529.855222688, + "particle_weight": 1.023645543155193e+31 }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 }, "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999366152e+29 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.0239999998887592e+29 }, - "alpha1": { - "particle_momentum_x": 4.665933695243743e-15, - "particle_momentum_y": 4.603805875733438e-15, - "particle_momentum_z": 4.706765986105302e-15, - "particle_position_x": 461871.79172011977, - "particle_position_y": 461162.2166206925, - "particle_position_z": 969262.7809050508, - "particle_weight": 1.9171578359058453e-27 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119663506409945e+31 + "alpha2": { + "particle_momentum_x": 4.1179548991012315e-15, + "particle_momentum_y": 4.110026665992801e-15, + "particle_momentum_z": 4.169802553223462e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 3.3372246639840338e+19 } -} \ No newline at end of file +} diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index a5549f30b6f..4f0d663388a 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -16,18 +16,15 @@ #include /** - * \brief Computes the total proton-boron fusion cross section in the range 0 < E < 3.5 MeV using - * the analytical fits given in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). - * For the record, note that there is a typo in equation (1) of this paper: the total cross section - * should read S(E)/E*exp(-sqrt(E_G/E)) instead of S(E)/E*exp(sqrt(E_G/E)) (minus sign in the - * exponential). + * \brief Computes the total proton-boron fusion cross section in the range 0 < E < 9.76 MeV using + * the analytical fits given in A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023). * * @param[in] E_keV the kinetic energy of the proton-boron pair in its center of mass frame, in * keV. * @return The total cross section in barn. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleReal& E_keV) +amrex::ParticleReal ProtonBoronFusionCrossSectionTentori (const amrex::ParticleReal& E_keV) { using namespace amrex::literals; using namespace amrex::Math; @@ -54,13 +51,13 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe // Compute astrophysical factor, in MeV barn, using the fits constexpr auto E_lim1 = 400._prt; // Limits between the different fit regions - constexpr auto E_lim2 = 642._prt; + constexpr auto E_lim2 = 668._prt; amrex::ParticleReal astrophysical_factor; if (E_keV < E_lim1) { constexpr auto C0 = 197._prt; - constexpr auto C1 = 0.24_prt; - constexpr auto C2 = 2.31e-4_prt; + constexpr auto C1 = 0.269_prt; + constexpr auto C2 = 2.54e-4_prt; constexpr auto AL = 1.82e4_prt; constexpr auto EL = 148._prt; constexpr auto dEL_sq = 2.35_prt*2.35_prt; @@ -69,28 +66,28 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe } else if (E_keV < E_lim2) { - constexpr auto D0 = 330._prt; - constexpr auto D1 = 66.1_prt; - constexpr auto D2 = -20.3_prt; - constexpr auto D5 = -1.58_prt; + constexpr auto D0 = 346._prt; + constexpr auto D1 = 150._prt; + constexpr auto D2 = -59.9_prt; + constexpr auto D5 = -0.460_prt; const amrex::ParticleReal E_norm = (E_keV-400._prt) * 1.e-2_prt; astrophysical_factor = D0 + D1*E_norm + D2*powi<2>(E_norm) + D5*powi<5>(E_norm); } else { - constexpr auto A0 = 2.57e6_prt; - constexpr auto A1 = 5.67e5_prt; - constexpr auto A2 = 1.34e5_prt; - constexpr auto A3 = 5.68e5_prt; - constexpr auto E0 = 581.3_prt; - constexpr auto E1 = 1083._prt; - constexpr auto E2 = 2405._prt; - constexpr auto E3 = 3344._prt; - constexpr auto dE0_sq = 85.7_prt*85.7_prt; - constexpr auto dE1_sq = 234._prt*234._prt; - constexpr auto dE2_sq = 138._prt*138._prt; - constexpr auto dE3_sq = 309._prt*309._prt; - constexpr auto B = 4.38_prt; + constexpr auto A0 = 1.98e6_prt; + constexpr auto A1 = 3.89e6_prt; + constexpr auto A2 = 1.36e6_prt; + constexpr auto A3 = 3.71e6_prt; + constexpr auto E0 = 640.9_prt; + constexpr auto E1 = 1211._prt; + constexpr auto E2 = 2340._prt; + constexpr auto E3 = 3294._prt; + constexpr auto dE0_sq = 85.5_prt*85.5_prt; + constexpr auto dE1_sq = 414._prt*414._prt; + constexpr auto dE2_sq = 221._prt*221._prt; + constexpr auto dE3_sq = 351._prt*351._prt; + constexpr auto B = 0.381_prt; astrophysical_factor = A0 / ((E_keV-E0)*(E_keV-E0) + dE0_sq) + A1 / ((E_keV-E1)*(E_keV-E1) + dE1_sq) + A2 / ((E_keV-E2)*(E_keV-E2) + dE2_sq) + @@ -102,10 +99,12 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe } /** - * \brief Computes the total proton-boron fusion cross section in the range E > 3.5 MeV using a + * \brief Computes the total proton-boron fusion cross section in the range E > 9.76 MeV using a * simple power law fit of the data presented in Buck et al., Nuclear Physics A, 398(2), 189-202 - * (1983) (data can also be found in the EXFOR database). - * + * (1983) (data can also be found in the EXFOR database). Note: the fit in Buck et al. started + * from 3.5 MeV. The same exponent power has been used here however the cross_section_start_fit + * has been modified to ensure exact continuity with the fit used for lower energies. + * @param[in] E_keV the kinetic energy of the proton-boron pair in its center of mass frame, in * keV. * @return The total cross section in barn. @@ -115,9 +114,9 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionBuck (const amrex::ParticleReal { using namespace amrex::literals; - constexpr amrex::ParticleReal E_start_fit = 3500._prt; // Fit starts at 3.5 MeV + constexpr amrex::ParticleReal E_start_fit = 9760._prt; // Fit starts at 9.76 MeV // cross section at E = E_start_fit, in barn - constexpr amrex::ParticleReal cross_section_start_fit = 0.2168440845211521_prt; + constexpr amrex::ParticleReal cross_section_start_fit = 0.01277998_prt; constexpr amrex::ParticleReal slope_fit = -2.661840717596765_prt; // Compute fitted value @@ -125,12 +124,13 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionBuck (const amrex::ParticleReal } /** - * \brief Computes the total proton-boron fusion cross section. When E_kin_star < 3.5 MeV, we use - * the analytical fits given in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). When - * E_kin_star > 3.5 MeV, we use a simple power law fit of the data presented in Buck et al., + * \brief Computes the total proton-boron fusion cross section using the analytical fit described + * in A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023). Includes the Breit-Wigner term + * to reconstruct the 148 keV resonance, which is missing from the Sikora, Wellar dataset. When + * E_kin_star > 9.76 MeV, we use a simple power law fit of the data presented in Buck et al., * Nuclear Physics A, 398(2), 189-202 (1983). Both fits return the same value for - * E_kin_star = 3.5 MeV. - * + * E_kin_star = 9.76 MeV. + * @param[in] E_kin_star the kinetic energy of the proton-boron pair in its center of mass frame, * in SI units. * @return The total cross section in SI units (square meters). @@ -143,10 +143,10 @@ amrex::ParticleReal ProtonBoronFusionCrossSection (const amrex::ParticleReal& E_ // Fits use energy in keV constexpr amrex::ParticleReal joule_to_keV = 1.e-3_prt/PhysConst::q_e; const amrex::ParticleReal E_keV = E_kin_star*joule_to_keV; - constexpr amrex::ParticleReal E_threshold = 3500._prt; + constexpr amrex::ParticleReal E_threshold = 9760._prt; const amrex::ParticleReal cross_section_b = (E_keV <= E_threshold) ? - ProtonBoronFusionCrossSectionNevins(E_keV) : + ProtonBoronFusionCrossSectionTentori(E_keV) : ProtonBoronFusionCrossSectionBuck(E_keV); // Convert cross section to SI units: barn to square meter From c55230bf6d56723372f8ad05b1cf8504dffba04b Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Thu, 16 May 2024 08:36:07 -0700 Subject: [PATCH 117/121] Avoid un-needed copy in BackTransformParticleFunctor (#4934) --- .../ComputeDiagFunctors/BackTransformParticleFunctor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index 8a6f0765664..9c2d10ad819 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -90,7 +90,7 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst const amrex::Real dt = warpx.getdt(0); for (WarpXParIter pti(*m_pc_src, lev); pti.isValid(); ++pti) { - auto ptile_dst = pc_dst.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex() ); + pc_dst.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex() ); } auto& particles = m_pc_src->GetParticles(lev); From 6d2c5edede06b176002d494fc475c45788f122e5 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sun, 19 May 2024 01:20:07 +0200 Subject: [PATCH 118/121] fix race conditions (#4932) --- Source/BoundaryConditions/WarpX_PEC.cpp | 6 +++--- .../Particles/PhysicalParticleContainer.cpp | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index a42873db555..0067f54d3ff 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -840,6 +840,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Each current component is handled separately below, starting with Jx. + grown_domain_box.convert(Jx_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -850,7 +851,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jx_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -875,6 +875,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Handle Jy. + grown_domain_box.convert(Jy_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -885,7 +886,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jy_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -910,6 +910,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Handle Jz. + grown_domain_box.convert(Jz_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -920,7 +921,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jz_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c7f44903d9b..3d134393f06 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1003,6 +1003,19 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const bool radially_weighted = plasma_injector.radially_weighted; #endif + + // User-defined integer and real attributes: prepare parsers + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); + for (int ia = 0; ia < n_user_int_attribs; ++ia) { + user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); + } + for (int ia = 0; ia < n_user_real_attribs; ++ia) { + user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); + } + MFItInfo info; if (do_tiling && Gpu::notInLaunchRegion()) { info.EnableTiling(tile_size); @@ -1156,19 +1169,13 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); for (int ia = 0; ia < n_user_int_attribs; ++ia) { pa_user_int_pinned[ia] = soa.GetIntData(particle_icomps[m_user_int_attribs[ia]]).data() + old_size; - user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); } for (int ia = 0; ia < n_user_real_attribs; ++ia) { pa_user_real_pinned[ia] = soa.GetRealData(particle_comps[m_user_real_attribs[ia]]).data() + old_size; - user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); } #ifdef AMREX_USE_GPU // To avoid using managed memory, we first define pinned memory vector, initialize on cpu, From 30a8c3e5b44db65489c5a0c002579c2ee5283c9f Mon Sep 17 00:00:00 2001 From: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Date: Mon, 20 May 2024 15:49:41 -0700 Subject: [PATCH 119/121] Ensure compilation with double precision particles for DSMC collisions (#4938) * ensure double precision particle with DSMC collisions * Check ParticleReal precision is double Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Avigdor Veksler Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp index e7a4eb722fc..ca0818a6041 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -7,6 +7,7 @@ * License: BSD-3-Clause-LBNL */ #include "DSMCFunc.H" +#include "Utils/TextMsg.H" /** * \brief Constructor of the DSMCFunc class @@ -22,6 +23,9 @@ DSMCFunc::DSMCFunc ( { using namespace amrex::literals; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (std::is_same::value), + "Particle precision must be double for DSMC collisions."); + const amrex::ParmParse pp_collision_name(collision_name); // query for a list of collision processes From 559842bd5ae4bda47c7db0a56151078ea5ebc389 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 20 May 2024 17:06:38 -0700 Subject: [PATCH 120/121] AMReX/pyAMReX/PICSAR: Weekly Update (#4940) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b54afde491b..de1c8237021 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - + cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 5e95dd04eb8..855dcff58fd 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 +branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index fcd35bbbe7d..ac321b8b694 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 +branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 146b6003729..349a4fd3f63 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86" +set(WarpX_amrex_branch "7ca419ebb90da60fefc01d8c1816846fff8638a5" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index f85af8727d4..79feddf3184 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "74d93b53d60b25200e36f345f7b0ac5d611512b9" +set(WarpX_pyamrex_branch "1b795c2c9741c8a63a362ceef8bd5e70cb242e92" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 470a497979b..099a59702a2 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - +cd amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 6c4171a6f4608cca4fa81cdd07f91b8ec3ae78a3 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Tue, 21 May 2024 15:42:15 -0400 Subject: [PATCH 121/121] Promote DSMC calculations to double precision to allow use with single-precision particles (#4941) * promote DSMC calculations to double precision * remove commented-out check * double -> auto for clang-tidy --- .../BinaryCollision/BinaryCollisionUtils.H | 62 ++++++++++--------- .../BinaryCollision/DSMC/DSMCFunc.cpp | 3 - 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 08ddcd27174..2e24a93a35e 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -62,60 +62,66 @@ namespace BinaryCollisionUtils{ using namespace amrex::literals; using namespace amrex::Math; - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); constexpr double c_sq = PhysConst::c * PhysConst::c; constexpr double inv_csq = 1.0 / c_sq; - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; + // Cast input parameters to double before computing collision properties + // This is needed to avoid errors when using single-precision particles + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const auto u1x_dbl = static_cast(u1x); + const auto u1y_dbl = static_cast(u1y); + const auto u1z_dbl = static_cast(u1z); + const auto u2x_dbl = static_cast(u2x); + const auto u2y_dbl = static_cast(u2y); + const auto u2z_dbl = static_cast(u2z); + + const double m1_sq = m1_dbl*m1_dbl; + const double m2_sq = m2_dbl*m2_dbl; // Compute Lorentz factor gamma in the lab frame - const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + const double g1 = std::sqrt( 1.0 + (u1x_dbl*u1x_dbl + u1y_dbl*u1y_dbl + u1z_dbl*u1z_dbl)*inv_csq ); + const double g2 = std::sqrt( 1.0 + (u2x_dbl*u2x_dbl + u2y_dbl*u2y_dbl + u2z_dbl*u2z_dbl)*inv_csq ); // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; + const double p1x = u1x_dbl * m1_dbl; + const double p1y = u1y_dbl * m1_dbl; + const double p1z = u1z_dbl * m1_dbl; + const double p2x = u2x_dbl * m2_dbl; + const double p2y = u2y_dbl * m2_dbl; + const double p2z = u2z_dbl * m2_dbl; + // Square norm of the total (sum between the two particles) momenta in the lab frame - const auto p_total_sq = static_cast( - powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) - ); + const double p_total_sq = powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z); // Total energy in the lab frame // Note the use of `double` for energy since this calculation is // prone to error with single precision. - const auto m1_dbl = static_cast(m1); - const auto m2_dbl = static_cast(m2); - const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + const double E_lab = (m1_dbl*g1 + m2_dbl*g2) * c_sq; // Total energy squared in the center of mass frame, calculated using the Lorentz invariance // of the four-momentum norm const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; // Kinetic energy in the center of mass frame const double E_star = std::sqrt(E_star_sq); + + // Cast back to chosen precision for output E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); // Square of the norm of the momentum of one of the particles in the center of mass frame // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle // The expression below is specifically written in a form that avoids returning // small negative numbers due to machine precision errors, for low-energy particles - const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); - const auto p_star_sq = static_cast( - m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) - ); + const double E_ratio = E_star/((m1_dbl + m2_dbl)*c_sq); + const double p_star_sq = m1_dbl*m2_dbl*c_sq * ( powi<2>(E_ratio) - 1.0 ) + + powi<2>(m1_dbl - m2_dbl)*c_sq/4.0 * powi<2>( E_ratio - 1.0/E_ratio); // Lorentz factors in the center of mass frame - const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); - const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + const double g1_star = std::sqrt(1.0 + p_star_sq / (m1_sq*c_sq)); + const double g2_star = std::sqrt(1.0 + p_star_sq / (m2_sq*c_sq)); - // relative velocity in the center of mass frame - v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + // relative velocity in the center of mass frame, cast back to chosen precision + v_rel_COM = static_cast(std::sqrt(p_star_sq) * (1.0/(m1_dbl*g1_star) + 1.0/(m2_dbl*g2_star))); // Cross sections and relative velocity are computed in the center of mass frame. // On the other hand, the particle densities (weight over volume) in the lab frame are used. @@ -124,7 +130,7 @@ namespace BinaryCollisionUtils{ // COM frame and the Lorentz factors in the lab frame (see // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor // is calculated here. - lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + lab_to_COM_lorentz_factor = static_cast(g1_star*g2_star/(g1*g2)); } /** diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp index ca0818a6041..9ee676bf002 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -23,9 +23,6 @@ DSMCFunc::DSMCFunc ( { using namespace amrex::literals; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (std::is_same::value), - "Particle precision must be double for DSMC collisions."); - const amrex::ParmParse pp_collision_name(collision_name); // query for a list of collision processes