From 2837f55d19cf1728751b788619911e44034bbe1e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 6 Aug 2024 14:01:35 -0700 Subject: [PATCH 01/17] Update test to use lasy 0.5.0 (#5111) --- Examples/Tests/laser_injection_from_file/inputs.1d_boost_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.1d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.2d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.3d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.RZ_test | 2 +- .../Tests/laser_injection_from_file/inputs.from_RZ_file_test | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test index adfde819531..ffc1865ee0f 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test @@ -47,7 +47,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_test b/Examples/Tests/laser_injection_from_file/inputs.1d_test index 6124b4552a2..6c392883418 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test b/Examples/Tests/laser_injection_from_file/inputs.2d_test index a3003a4978d..e5814471753 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.2d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.3d_test b/Examples/Tests/laser_injection_from_file/inputs.3d_test index e3096bf92d9..ad8159cb650 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.3d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.3d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.RZ_test b/Examples/Tests/laser_injection_from_file/inputs.RZ_test index fe5bb675212..2a539883fec 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.RZ_test +++ b/Examples/Tests/laser_injection_from_file/inputs.RZ_test @@ -41,7 +41,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test index ca50430c21a..f92440188b7 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test +++ b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test @@ -40,7 +40,7 @@ lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_RZ_laser.profile = from_file lasy_RZ_laser.time_chunk_size = 50 -lasy_RZ_laser.lasy_file_name = "laguerrelaserRZ_00000.h5" +lasy_RZ_laser.lasy_file_name = "diags/laguerrelaserRZ_00000.h5" lasy_RZ_laser.delay = 0.0 # Diagnostics From 53605cd77f0c3478065dec270ca6bf5955795e2f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 7 Aug 2024 08:56:11 -0700 Subject: [PATCH 02/17] Fix bug in `PrintMainPICparameters` (uninitialized values) (#5114) --- Source/WarpX.H | 2 +- Source/WarpX.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index ad6e8a3abfd..da60824945e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -233,7 +233,7 @@ public: //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation - bool current_correction; + bool current_correction = true; //! If true, the PSATD update equation for E contains both J and rho //! (default is false for standard PSATD and true for Galilean PSATD) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cd86587f2d8..4d51dd283a3 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1544,7 +1544,6 @@ WarpX::ReadParameters () // Current correction activated by default, unless a charge-conserving // current deposition (Esirkepov, Vay) or the div(E) cleaning scheme // are used - current_correction = true; if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor || WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay || From db2fbcda27c318a87a4de3c6ce4ae62904a2ef42 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 7 Aug 2024 10:59:17 -0500 Subject: [PATCH 03/17] Temporarily disable `openbc_poisson_solver` (#5117) This test uncoveres a bug of an invalid pc passed early to I/O routines. We will add it back with the fix. --- .../{inputs_3d => README} | 0 Regression/WarpX-tests.ini | 26 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) rename Examples/Tests/openbc_poisson_solver/{inputs_3d => README} (100%) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/README similarity index 100% rename from Examples/Tests/openbc_poisson_solver/inputs_3d rename to Examples/Tests/openbc_poisson_solver/README diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 96dbea5c380..bc470d7706b 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3952,16 +3952,16 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py -[openbc_poisson_solver] -buildDir = . -inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py +#[openbc_poisson_solver] +#buildDir = . +#inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +#runtime_params = warpx.abort_on_warning_threshold = high +#dim = 3 +#addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +#cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON +#restartTest = 0 +#useMPI = 1 +#numprocs = 2 +#useOMP = 1 +#numthreads = 1 +#analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py From ffb16f3509b166d1a31b90b81c5bd19e24f54c7a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 7 Aug 2024 13:09:51 -0700 Subject: [PATCH 04/17] CI: Clean out Pip Caches (#5110) * Debugging CI tests * Run all Cartesian 3D tests * Clear cache after python installation commands * Update Regression/WarpX-tests.ini * `run_test.sh`: Detailed `df -h` Prints * Update checksum * Cleanup --------- Co-authored-by: Remi Lehe Co-authored-by: Axel Huebl --- .../{README => inputs_3d} | 0 .../benchmarks_json/collisionXYZ.json | 28 +++++++++---------- Regression/WarpX-tests.ini | 26 ++++++++--------- run_test.sh | 1 + 4 files changed, 28 insertions(+), 27 deletions(-) rename Examples/Tests/openbc_poisson_solver/{README => inputs_3d} (100%) diff --git a/Examples/Tests/openbc_poisson_solver/README b/Examples/Tests/openbc_poisson_solver/inputs_3d similarity index 100% rename from Examples/Tests/openbc_poisson_solver/README rename to Examples/Tests/openbc_poisson_solver/inputs_3d diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index c13f404857a..726573ef8b2 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 381957.7394223898, - "T_ion": 319565.6269784763 + "T_electron": 351982.0169218243, + "T_ion": 349599.6939052666 }, "electron": { - "particle_momentum_x": 8.631833485301232e-19, - "particle_momentum_y": 8.476316745254719e-19, - "particle_momentum_z": 8.514139891331418e-19, - "particle_position_x": 21253105.73686561, - "particle_position_y": 21282643.519070115, - "particle_position_z": 21239057.457948968, + "particle_momentum_x": 8.359982321196841e-19, + "particle_momentum_y": 8.192841151167721e-19, + "particle_momentum_z": 8.182985690701241e-19, + "particle_position_x": 21255110.08090505, + "particle_position_y": 21303488.6242626, + "particle_position_z": 21238676.122703437, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 1.9215585867122464e-18, - "particle_momentum_y": 1.7471481568315848e-18, - "particle_momentum_z": 1.7510887207292533e-18, - "particle_position_x": 21228900.948879313, - "particle_position_y": 21304564.011731848, - "particle_position_z": 21250585.221808463, + "particle_momentum_x": 2.0034830240966893e-18, + "particle_momentum_y": 1.8323959076577197e-18, + "particle_momentum_z": 1.827953230828629e-18, + "particle_position_x": 21246214.748882487, + "particle_position_y": 21280709.710960124, + "particle_position_z": 21206153.002106402, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index bc470d7706b..96dbea5c380 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3952,16 +3952,16 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py -#[openbc_poisson_solver] -#buildDir = . -#inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -#runtime_params = warpx.abort_on_warning_threshold = high -#dim = 3 -#addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -#cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -#restartTest = 0 -#useMPI = 1 -#numprocs = 2 -#useOMP = 1 -#numthreads = 1 -#analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py +[openbc_poisson_solver] +buildDir = . +inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/run_test.sh b/run_test.sh index f56a957c17a..4efa86eb36c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -65,6 +65,7 @@ python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade cmake python3 -m pip install --upgrade -r warpx/Regression/requirements.txt +python3 -m pip cache purge # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git From fe36717b3366a1a564f41cdc152f2d6b6660a450 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 7 Aug 2024 17:34:29 -0500 Subject: [PATCH 05/17] Fix: `species_type` init memory (#5119) The `PC::physical_species` was undefined memory unless `.species_type` was provided in the input set. This initializes this member variable to a clear `PhysicalSpecies::unspecified` instead of random memory, which caused UB in functions such as `IamA<...>` during I/O. --- Source/Particles/WarpXParticleContainer.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index d4c325fbfb8..b44cb33d66b 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -433,7 +433,7 @@ protected: amrex::ParticleReal charge; amrex::ParticleReal mass; - PhysicalSpecies physical_species; + PhysicalSpecies physical_species = PhysicalSpecies::unspecified; // Controls boundaries for particles exiting the domain ParticleBoundaries m_boundary_conditions; From 547794da10a10698bb9fdd434b2935ba063521a6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 7 Aug 2024 17:36:04 -0500 Subject: [PATCH 06/17] AnyFFT: 1D Support (#5112) * AnyFFT 1D: FFTW * AnyFFT 1D: CuFFT --- Source/ablastr/math/fft/WrapCuFFT.cpp | 9 ++++++++- Source/ablastr/math/fft/WrapFFTW.cpp | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index 9ceb91457c9..07a2d9d2732 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -42,8 +42,11 @@ namespace ablastr::math::anyfft } else if (dim == 2) { result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorR2C); + } else if (dim == 1) { + result = cufftPlan1d( + &(fft_plan.m_plan), real_size[0], VendorR2C, 1); } else { - ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); + ABLASTR_ABORT_WITH_MESSAGE("only dim=1 and dim=2 and dim=3 have been implemented"); } } else { if (dim == 3) { @@ -52,6 +55,10 @@ namespace ablastr::math::anyfft } else if (dim == 2) { result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorC2R); + } else if (dim == 1) { + int batch = 2; + result = cufftPlan1d( + &(fft_plan.m_plan), real_size[0], VendorC2R, 1); } else { ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); } diff --git a/Source/ablastr/math/fft/WrapFFTW.cpp b/Source/ablastr/math/fft/WrapFFTW.cpp index 6711bbface9..16f0355cc5d 100644 --- a/Source/ablastr/math/fft/WrapFFTW.cpp +++ b/Source/ablastr/math/fft/WrapFFTW.cpp @@ -25,11 +25,15 @@ namespace ablastr::math::anyfft const auto VendorCreatePlanC2R3D = fftwf_plan_dft_c2r_3d; const auto VendorCreatePlanR2C2D = fftwf_plan_dft_r2c_2d; const auto VendorCreatePlanC2R2D = fftwf_plan_dft_c2r_2d; + const auto VendorCreatePlanR2C1D = fftwf_plan_dft_r2c_1d; + const auto VendorCreatePlanC2R1D = fftwf_plan_dft_c2r_1d; #else const auto VendorCreatePlanR2C3D = fftw_plan_dft_r2c_3d; const auto VendorCreatePlanC2R3D = fftw_plan_dft_c2r_3d; const auto VendorCreatePlanR2C2D = fftw_plan_dft_r2c_2d; const auto VendorCreatePlanC2R2D = fftw_plan_dft_c2r_2d; + const auto VendorCreatePlanR2C1D = fftw_plan_dft_r2c_1d; + const auto VendorCreatePlanC2R1D = fftw_plan_dft_c2r_1d; #endif FFTplan CreatePlan(const amrex::IntVect& real_size, amrex::Real * const real_array, @@ -56,9 +60,12 @@ namespace ablastr::math::anyfft } else if (dim == 2) { fft_plan.m_plan = VendorCreatePlanR2C2D( real_size[1], real_size[0], real_array, complex_array, FFTW_ESTIMATE); + } else if (dim == 1) { + fft_plan.m_plan = VendorCreatePlanR2C1D( + real_size[0], real_array, complex_array, FFTW_ESTIMATE); } else { ABLASTR_ABORT_WITH_MESSAGE( - "only dim=2 and dim=3 have been implemented"); + "only dim=1 and dim=2 and dim=3 have been implemented"); } } else if (dir == direction::C2R){ if (dim == 3) { @@ -67,9 +74,12 @@ namespace ablastr::math::anyfft } else if (dim == 2) { fft_plan.m_plan = VendorCreatePlanC2R2D( real_size[1], real_size[0], complex_array, real_array, FFTW_ESTIMATE); + } else if (dim == 1) { + fft_plan.m_plan = VendorCreatePlanC2R1D( + real_size[0], complex_array, real_array, FFTW_ESTIMATE); } else { ABLASTR_ABORT_WITH_MESSAGE( - "only dim=2 and dim=3 have been implemented. Should be easy to add dim=1."); + "only dim=1 and dim=2 and dim=3 have been implemented."); } } From 7d409630e7edac0f041e5c1f56a02e81b7561422 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 8 Aug 2024 09:22:40 -0700 Subject: [PATCH 07/17] CI: clean test dirs, reset `collisionXZ` checksums (#5120) * CI: add option to clean up test dirs * Reset Checksum: `collisionXZ` * Env Var Control: `WARPX_CI_CLEAN_TESTS` --------- Co-authored-by: Axel Huebl --- .azure-pipelines.yml | 1 + .../Checksum/benchmarks_json/collisionXZ.json | 24 +++++++++---------- run_test.sh | 16 +++++++++---- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7e4edc2dc36..7c71599bbf3 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -18,6 +18,7 @@ jobs: LAPACKPP_HOME: '/usr/local' OMP_NUM_THREADS: 1 WARPX_CI_CCACHE: 'TRUE' + WARPX_CI_CLEAN_TESTS: 'TRUE' WARPX_CI_NUM_MAKE_JOBS: 2 WARPX_CI_OPENPMD: 'TRUE' WARPX_CI_TMP: '/tmp/ci' diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index f90c34bc86d..4fcf00ced62 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -7,20 +7,20 @@ "Ey": 0.0, "Ez": 0.0 }, - "ion": { - "particle_momentum_x": 2.4932317055825563e-19, - "particle_momentum_y": 2.274916403334278e-19, - "particle_momentum_z": 2.2528161767665816e-19, - "particle_position_x": 2648815.601036139, - "particle_position_y": 2662836.7581390506, + "electron": { + "particle_momentum_x": 1.0618161248729303e-19, + "particle_momentum_y": 1.0331186403682394e-19, + "particle_momentum_z": 1.0375409035564829e-19, + "particle_position_x": 2652982.4592067804, + "particle_position_y": 2666143.4238272114, "particle_weight": 1.7256099431746894e+26 }, - "electron": { - "particle_momentum_x": 1.0489203687862582e-19, - "particle_momentum_y": 1.0209657029567292e-19, - "particle_momentum_z": 1.0248962872393911e-19, - "particle_position_x": 2657004.8285825616, - "particle_position_y": 2670174.272797987, + "ion": { + "particle_momentum_x": 2.4479519290953386e-19, + "particle_momentum_y": 2.2313460312794214e-19, + "particle_momentum_z": 2.207395147435577e-19, + "particle_position_x": 2666525.886108531, + "particle_position_y": 2666683.4040517565, "particle_weight": 1.7256099431746894e+26 } } diff --git a/run_test.sh b/run_test.sh index 4efa86eb36c..275b7b5efb0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -16,8 +16,10 @@ # physically correct. # The tests can be influenced by environment variables: +# Use `export WARPX_CI_CLEAN_TESTS=ON` in order to remove all subdirectories +# from each test directory, directly after a test has passed. # Use `export WARPX_CI_DIM=3` or `export WARPX_CI_DIM=2` in order to -# select only the tests that correspond to this dimension +# select only the tests that correspond to this dimension. # Use `export WARPX_TEST_ARCH=CPU` or `export WARPX_TEST_ARCH=GPU` in order # to run the tests on CPU or GPU respectively. @@ -30,6 +32,7 @@ tests_arg=$* tests_run=${tests_arg:+--tests=${tests_arg}} # environment options +WARPX_CI_CLEAN_TESTS=${WARPX_CI_CLEAN_TESTS:-""} WARPX_CI_TMP=${WARPX_CI_TMP:-""} # Remove contents and link to a previous test directory (intentionally two arguments) @@ -80,7 +83,7 @@ curl -sOL https://github.com/openPMD/openPMD-example-datasets/raw/4ba1d257c5b489 cd - # Clone the AMReX regression test utility -git clone https://github.com/AMReX-Codes/regression_testing.git +git clone -b EZoni_rm_testdir https://github.com/EZoni/regression_testing.git # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks @@ -93,12 +96,17 @@ cp -r Checksum ../../regression_testing/ # Run tests cd ../../regression_testing/ echo "cd $PWD" +if [ -z "${WARPX_CI_CLEAN_TESTS}" ]; then + test_rm_dir="" +else + test_rm_dir="--rm_testdir" +fi # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then - python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all "${tests_run}" + python3 regtest.py ../rt-WarpX/ci-tests.ini ${test_rm_dir} --skip_comparison --no_update all "${tests_run}" # run all tests (variables tests_arg and tests_run are empty) else - python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all + python3 regtest.py ../rt-WarpX/ci-tests.ini ${test_rm_dir} --skip_comparison --no_update all fi # clean up python virtual environment From 13fa090d9d51072b4ded96a6a93fec4996be46b4 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:29:37 -0700 Subject: [PATCH 08/17] Add density min and max arguments to `picmi.AnalyticDistribution` (#5108) * add density min and max arguments to `picmi.AnalyticDistribution` * add docstring --- Python/pywarpx/picmi.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 8be6d9c6212..a7ef3470c52 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -504,6 +504,11 @@ def set_species_attributes(self, species, layout, source_name): species.add_new_group_attr(source_name, 'uy', self.directed_velocity[1]/constants.c) species.add_new_group_attr(source_name, 'uz', self.directed_velocity[2]/constants.c) + if hasattr(self, 'density_min'): + species.add_new_group_attr(source_name, 'density_min', self.density_min) + if hasattr(self, 'density_max'): + species.add_new_group_attr(source_name, 'density_max', self.density_max) + def setup_parse_momentum_functions(self, species, source_name, expressions, suffix, defaults): for sdir, idir in zip(['x', 'y', 'z'], [0, 1, 2]): if expressions[idir] is not None: @@ -555,6 +560,14 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist Parameters ---------- + warpx_density_min: float + Minimum plasma density. No particle is injected where the density is + below this value. + + warpx_density_max: float + Maximum plasma density. The density at each point is the minimum between + the value given in the profile, and density_max. + warpx_momentum_spread_expressions: list of string Analytic expressions describing the gamma*velocity spread for each axis [m/s]. Expressions should be in terms of the position, written as 'x', 'y', and 'z'. @@ -563,6 +576,8 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist """ def init(self, kw): + self.density_min = kw.pop('warpx_density_min', None) + self.density_max = kw.pop('warpx_density_max', None) self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): From 4b04701fb1bbe900ff0d56e3e2c14006521d0b20 Mon Sep 17 00:00:00 2001 From: Daniel <92740330+PhysicsDan@users.noreply.github.com> Date: Fri, 9 Aug 2024 23:32:49 +0100 Subject: [PATCH 09/17] Change alpha particle momentum initialization (#5033) * Change alpha particle momentum initialization Now alpha particles will be initializes with momentum calculated via the two step alpha_1 channel, which has a higher cross section than the alpha_0 channel currently used. * Fix: Corrected Be mass Corrected the mass of the Be to account for the mass defect due to being an excited state. The conservation of energy tests now pass. * Edit fusion and decay energy * Modified released in fusion and decay steps of pB reaction. Ensures the delta mc2 = energy released, at both the fusion and decay step * Modified tolerence in pB fusion test * Test one - abs. tol. in min energy of alpha2 and alpha3 is 20 keV * Test two - abs. tol. in min energy of alpha2 and alpha3 is 200 keV * Update checksum for tests * Updated alpha particle energy checks * Revert max/min energy check test In `check_initial_energy2` the max and min sim. values are above and below theoretical calculations. Tests now check variation is within an absolute tolerance. Note: Total energy is still conserved. * Increased atol for check_initial_energy2 * Changed max alpha particle energy check Now the maximum alpha particle energy from simulation is compared to the analytical value using relative tolerance, not abs tolerance. * Fixed type --- .../analysis_proton_boron_fusion.py | 32 +-- .../Proton_Boron_Fusion_2D.json | 138 ++++++------- .../Proton_Boron_Fusion_3D.json | 183 +++++++++--------- .../ProtonBoronFusionInitializeMomentum.H | 21 +- 4 files changed, 192 insertions(+), 182 deletions(-) diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index 543eb62484a..ea0f323f722 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -68,15 +68,15 @@ m_p = 1.00782503223*scc.m_u # Proton mass m_b = 11.00930536*scc.m_u # Boron 11 mass m_reduced = m_p*m_b/(m_p+m_b) -m_a = 4.00260325413*scc.m_u # Alpha mass -m_be = 7.94748*scc.m_p # Beryllium 8 mass +m_a = 4.00260325413*scc.m_u # Alpha (He4) mass +m_be = (8.0053095729+0.00325283863)*scc.m_u # Be8* mass (3.03 MeV ex. state) Z_boron = 5. Z_proton = 1. E_Gamow = (Z_boron*Z_proton*np.pi*scc.fine_structure)**2*2.*m_reduced*scc.c**2 E_Gamow_MeV = E_Gamow/MeV_to_Joule E_Gamow_keV = E_Gamow/keV_to_Joule -E_fusion = 8.59009*MeV_to_Joule # Energy released during p + B -> alpha + Be -E_decay = 0.0918984*MeV_to_Joule # Energy released during Be -> 2*alpha +E_fusion = 5.55610759*MeV_to_Joule # Energy released during p + B -> alpha + Be* +E_decay = 3.12600414*MeV_to_Joule # Energy released during Be* -> 2*alpha E_fusion_total = E_fusion + E_decay # Energy released during p + B -> 3*alpha ## Checks whether this is the 2D or the 3D test @@ -493,10 +493,13 @@ def check_initial_energy1(data, E_com): energy_alpha3_simulation = energy_alpha_slice[4::6] assert(np.all(is_close(energy_alpha1_simulation, energy_alpha1_theory, rtol=5.e-8))) - assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, rtol=1.e-2)) + ## Check that the max / min value are comparable to the analytical value + ## The minimum value is checked to be within 20 keV of the analytical value + ## The maximum value is checked to be within 1% of the analytical value + assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=1.e-2 )) + assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, atol=3.218e-15 )) + assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=1.e-2 )) + assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, atol=3.218e-15 )) def check_initial_energy2(data): ## In WarpX, the initial momentum of the alphas is computed assuming that the fusion process @@ -578,13 +581,16 @@ def check_initial_energy2(data): assert(is_close(np.amax(energy_alpha1_simulation), max_energy_alpha1, rtol=1.e-2)) assert(is_close(np.amin(energy_alpha1_simulation), min_energy_alpha1, rtol=1.e-2)) - ## Tolerance is quite high below because we don't have a lot of alphas to produce good + ## Check that the max / min value are comparable to the analytical value + ## The minimum value is checked to be within 200 keV of the analytical value + ## The maximum value is checked to be within 5% of the analytical value + ## Tolerance is quite high because we don't have a lot of alphas to produce good ## statistics and an event like alpha1 emitted exactly in direction of proton & alpha2 ## emitted exactly in direction opposite to Beryllium is somewhat rare. - assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, rtol=2.5e-1)) + assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=5e-2 )) + assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, atol=3.218e-14 )) + assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=5e-2 )) + assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, atol=3.218e-14 )) def check_xy_isotropy(data): ## Checks that the alpha particles are emitted isotropically in x and y diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index cdee4b078b0..cf70a750b44 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -2,22 +2,6 @@ "lev=0": { "rho": 0.0 }, - "alpha2": { - "particle_momentum_x": 4.057984510682467e-15, - "particle_momentum_y": 4.104188139725855e-15, - "particle_momentum_z": 4.17858000090827e-15, - "particle_position_x": 408793.7905852193, - "particle_position_y": 861780.8020495367, - "particle_weight": 5.078061191951185e+18 - }, - "alpha3": { - "particle_momentum_x": 5.017656304003558e-16, - "particle_momentum_y": 4.935595075276182e-16, - "particle_momentum_z": 4.867133212376827e-16, - "particle_position_x": 52678.192400911765, - "particle_position_y": 105483.59950020742, - "particle_weight": 1.5413633830148085e+27 - }, "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -26,6 +10,22 @@ "particle_position_y": 81921136.14476715, "particle_weight": 128.00000000000261 }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399486212205656e+30 + }, + "alpha5": { + "particle_momentum_x": 2.3827075626988002e-14, + "particle_momentum_y": 2.388035811027714e-14, + "particle_momentum_z": 2.4040151761604078e-14, + "particle_position_x": 2457556.8571638423, + "particle_position_y": 4914659.635379322, + "particle_weight": 3.839999999999998e-19 + }, "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -34,53 +34,37 @@ "particle_position_y": 819270.9858143466, "particle_weight": 1.2799999998307316e+28 }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 - }, "alpha1": { - "particle_momentum_x": 4.691909092431811e-15, - "particle_momentum_y": 4.6836958163275755e-15, - "particle_momentum_z": 4.657203546376977e-15, + "particle_momentum_x": 4.758865177822356e-15, + "particle_momentum_y": 4.7625215232714745e-15, + "particle_momentum_z": 4.734412976565431e-15, "particle_position_x": 463465.27131109464, "particle_position_y": 978207.6061359186, "particle_weight": 4.977661460251435e-28 }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512788807322e+28 - }, - "proton1": { + "proton5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 }, - "alpha5": { - "particle_momentum_x": 2.3343485859070736e-14, - "particle_momentum_y": 2.3451128753701046e-14, - "particle_momentum_z": 2.3579462789062662e-14, - "particle_position_x": 2457556.8571638423, - "particle_position_y": 4914659.635379322, - "particle_weight": 3.839999999999998e-19 + "proton3": { + "particle_momentum_x": 1.6847290893972251e-15, + "particle_momentum_y": 1.6827074502304075e-15, + "particle_momentum_z": 1.6802489646490975e-15, + "particle_position_x": 2457270.6999197667, + "particle_position_y": 4914315.665267942, + "particle_weight": 1.279486212205663e+30 }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399486212205656e+30 + "alpha2": { + "particle_momentum_x": 4.140109871048478e-15, + "particle_momentum_y": 4.159837925039718e-15, + "particle_momentum_z": 4.251680389225861e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 5.078061191951185e+18 }, "proton4": { "particle_momentum_x": 0.0, @@ -90,28 +74,44 @@ "particle_position_y": 819198.7077771134, "particle_weight": 1.2800000000000004e+37 }, - "proton3": { - "particle_momentum_x": 1.6847290893972251e-15, - "particle_momentum_y": 1.6827074502304075e-15, - "particle_momentum_z": 1.6802489646490975e-15, - "particle_position_x": 2457270.6999197667, - "particle_position_y": 4914315.665267942, - "particle_weight": 1.279486212205663e+30 + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, + "particle_weight": 128.00000000000261 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 }, "alpha4": { - "particle_momentum_x": 2.338084461204216e-14, - "particle_momentum_y": 2.3436156778849828e-14, - "particle_momentum_z": 2.35535708386288e-14, + "particle_momentum_x": 2.3854178157605226e-14, + "particle_momentum_y": 2.3882457581100904e-14, + "particle_momentum_z": 2.403451456378969e-14, "particle_position_x": 2457367.4582781536, "particle_position_y": 4915112.044373056, "particle_weight": 384.0000000000002 }, - "proton5": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512788807322e+28 + }, + "alpha3": { + "particle_momentum_x": 5.079136896829917e-16, + "particle_momentum_y": 5.075922501147803e-16, + "particle_momentum_z": 4.962151258214859e-16, + "particle_position_x": 52678.192400911765, + "particle_position_y": 105483.59950020742, + "particle_weight": 1.5413633830148085e+27 } } diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index ec7e047c537..5aa161dab99 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,17 +1,17 @@ { - "lev=0": { - "rho": 0.0 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "boron1": { + "lev=0": { + "rho": 0.0 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 + }, + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.524872467113344e-13, @@ -19,35 +19,36 @@ "particle_position_y": 40961136.14476712, "particle_position_z": 81920546.19181262, "particle_weight": 1024.000000000021 - }, - "alpha3": { - "particle_momentum_x": 4.764404554793872e-16, - "particle_momentum_y": 4.655900875811434e-16, - "particle_momentum_z": 4.578927372510084e-16, - "particle_position_x": 50987.442011759704, - "particle_position_y": 48999.674675246955, - "particle_position_z": 101142.57224226737, - "particle_weight": 1.0633705344227063e+28 - }, - "alpha1": { - "particle_momentum_x": 4.665933695243743e-15, - "particle_momentum_y": 4.603805875733438e-15, - "particle_momentum_z": 4.706765986105302e-15, + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.1196455431551905e+31 + }, + "alpha2": { + "particle_momentum_x": 4.172264972648105e-15, + "particle_momentum_y": 4.1838781269703316e-15, + "particle_momentum_z": 4.256769380749126e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 3.3372246639840338e+19 + }, + + "alpha1": { + "particle_momentum_x": 4.743550273664842e-15, + "particle_momentum_y": 4.703189539479841e-15, + "particle_momentum_z": 4.75829084832661e-15, "particle_position_x": 461871.79172011977, "particle_position_y": 461162.2166206925, "particle_position_z": 969262.7809050508, "particle_weight": 3.2387855108185994e-27 - }, - "alpha5": { - "particle_momentum_x": 2.3388206254864998e-14, - "particle_momentum_y": 2.334372885765467e-14, - "particle_momentum_z": 2.363588638941874e-14, - "particle_position_x": 2457556.857163843, - "particle_position_y": 2457059.6353793247, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron5": { + }, + "boron5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, @@ -55,17 +56,17 @@ "particle_position_y": 409518.5558814353, "particle_position_z": 819306.5006950963, "particle_weight": 1023.9999999999999 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3745333755307162e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.022781024024315e+29 - }, - "proton4": { + }, + "proton3": { + "particle_momentum_x": 1.6847282386883186e-15, + "particle_momentum_y": 1.6828065767793222e-15, + "particle_momentum_z": 1.6803456707569493e-15, + "particle_position_x": 2457343.371083716, + "particle_position_y": 2457033.3891170574, + "particle_position_z": 4914529.855222688, + "particle_weight": 1.023645543155193e+31 + }, + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 1.7586062624930794e-15, @@ -73,44 +74,44 @@ "particle_position_y": 409598.7077771135, "particle_position_z": 818958.0399127571, "particle_weight": 1.0240000000000003e+38 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.1196455431551905e+31 - }, - "alpha4": { - "particle_momentum_x": 2.33898275612641e-14, - "particle_momentum_y": 2.3423797451957437e-14, - "particle_momentum_z": 2.3516107929259732e-14, + }, + "alpha3": { + "particle_momentum_x": 4.822525301678111e-16, + "particle_momentum_y": 4.78793407068675e-16, + "particle_momentum_z": 4.710411155693663e-16, + "particle_position_x": 50987.442011759704, + "particle_position_y": 48999.674675246955, + "particle_position_z": 101142.57224226737, + "particle_weight": 1.0633705344227063e+28 + }, + "alpha5": { + "particle_momentum_x": 2.3856918226484716e-14, + "particle_momentum_y": 2.3827536503955812e-14, + "particle_momentum_z": 2.405465166862308e-14, + "particle_position_x": 2457556.857163843, + "particle_position_y": 2457059.6353793247, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 + }, + "alpha4": { + "particle_momentum_x": 2.386438069023501e-14, + "particle_momentum_y": 2.3877492448396915e-14, + "particle_momentum_z": 2.4013650859080414e-14, "particle_position_x": 2457367.458278153, "particle_position_y": 2457512.0443730573, "particle_position_z": 4914475.7765130745, "particle_weight": 3072.000000000002 - }, - "proton3": { - "particle_momentum_x": 1.6847282386883186e-15, - "particle_momentum_y": 1.6828065767793222e-15, - "particle_momentum_z": 1.6803456707569493e-15, - "particle_position_x": 2457343.371083716, - "particle_position_y": 2457033.3891170574, - "particle_position_z": 4914529.855222688, - "particle_weight": 1.023645543155193e+31 - }, - "proton5": { + }, + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 - }, - "boron2": { + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.022781024024315e+29 + }, + "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, @@ -118,14 +119,14 @@ "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, "particle_weight": 1.0239999998887592e+29 - }, - "alpha2": { - "particle_momentum_x": 4.1179548991012315e-15, - "particle_momentum_y": 4.110026665992801e-15, - "particle_momentum_z": 4.169802553223462e-15, - "particle_position_x": 408575.75269073684, - "particle_position_y": 413407.5155277014, - "particle_position_z": 863983.4313441743, - "particle_weight": 3.3372246639840338e+19 - } + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 + } } diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 375cc1e6d51..63f35d3d254 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -33,11 +33,11 @@ namespace { * \brief This function initializes the momentum of the alpha particles produced from * proton-boron fusion. The momentum is initialized by assuming that the fusion of a proton * with a boron nucleus into 3 alphas takes place in two steps. In the first step, the proton - * and the boron fuse into a beryllium nucleus and an alpha particle. In the second step, the - * beryllium decays into two alpha particles. The first step produces 8.59009 MeV of kinetic - * energy while the second step produces 91.8984 keV of kinetic energy. This two-step process + * and the boron fuse into an excited beryllium nucleus and an alpha particle. In the second step, the + * excited beryllium decays into two alpha particles. The first step produces ~5.56 MeV of kinetic + * energy while the second step produces ~3.12 MeV of kinetic energy. This two-step process * is considered to be the dominant process of proton+boron fusion into alphas (see - * Becker et al., Zeitschrift für Physik A Atomic Nuclei, 327(3), 341-355 (1987)). + * D.R. Tilley et al. / Nuclear Physics A 745 (2004) 155–362). * For each step, we assume in this function that the particles are emitted isotropically in * the corresponding center of mass frame (center of mass frame of proton + boron for the * creation of first alpha+beryllium and rest frame of beryllium for the creation of second and @@ -73,18 +73,21 @@ namespace { constexpr amrex::ParticleReal mev_to_joule = PhysConst::q_e*1.e6_prt; // Energy produced in the fusion reaction proton + boron11 -> Beryllium8 + alpha - // cf. Janis book of proton-induced cross-sections (2019) - constexpr amrex::ParticleReal E_fusion = 8.59009_prt*mev_to_joule; + // cf. https://doi.org/10.1016/j.radphyschem.2022.110727 + // Dominant reaction channel is p + B11 -> alpha_1 + 8Be* -> alpha_1 + alpha_11 + alpha_12 + 8.68 MeV + // cf. Kelley et al., (2017) http://dx.doi.org/10.1016/j.nuclphysa.2017.07.015 + constexpr amrex::ParticleReal E_fusion = 5.55610759_prt*mev_to_joule; // Energy produced when Beryllium8 decays into two alphas - // cf. JEFF-3.3 radioactive decay data library (2017) - constexpr amrex::ParticleReal E_decay = 0.0918984_prt*mev_to_joule; + // cf. Kelley et al., (2017) http://dx.doi.org/10.1016/j.nuclphysa.2017.07.015 + constexpr amrex::ParticleReal E_decay = 3.12600414_prt*mev_to_joule; // The constexprs ma_sq and mBe_sq underflow in single precision because we use SI units, // which can cause compilation to fail or generate a warning, so we're explicitly setting // them as double. Note that nuclear fusion module does not currently work with single // precision anyways. constexpr double m_alpha = PhysConst::m_u * 4.00260325413_prt; - constexpr double m_beryllium = PhysConst::m_p * 7.94748_prt; + // mass of the Be8 excited state (3.03 MeV above ground state) + constexpr double m_beryllium = PhysConst::m_u*(8.0053095729_prt+0.00325283863_prt); constexpr double mBe_sq = m_beryllium*m_beryllium; constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; From e246300d8289a5b37c9ea4ae00e501a6294e8909 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 9 Aug 2024 17:50:51 -0500 Subject: [PATCH 10/17] Checksum Reset via Env Variable (#5105) * Checksum Reset via Env Variable In preparation for CTest adoption, we want to make it easier to quickly run and reset checksums locally. This adds the feature to reset instead of evaluate in regular test runs. * Improve wording Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Docs/source/developers/checksum.rst | 15 +++++++++++++++ Regression/Checksum/checksumAPI.py | 17 ++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 81d16809d50..2452d074ba1 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -65,6 +65,21 @@ Since this will automatically change the JSON file stored on the repo, make a se git add .json git commit -m "reset benchmark for because ..." --author="Tools " +Automated reset of a list of test benchmarks +-------------------------------------------- + +If you set the environment variable ``export CHECKSUM_RESET=ON`` before running tests that are compared against existing benchmarks, the test analysis will reset the benchmarks to the new values, skipping the comparison. + +With `CTest `__ (coming soon), select the test(s) to reset by `name `__ or `label `__. + +.. code-block:: bash + + # regex filter: matched names + CHECKSUM_RESET=ON ctest --test-dir build -R "Langmuir_multi|LaserAcceleration" + + # ... check and commit changes ... + + Reset a benchmark from the Azure pipeline output on Github ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index cc13ceefa28..11adae5b5e1 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -42,6 +42,9 @@ def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. + If the environment variable CHECKSUM_RESET is set while this function is run, + the evaluation will be replaced with a call to reset_benchmark (see below). + Parameters ---------- test_name: string @@ -65,9 +68,17 @@ def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, output_file, output_format, - do_fields=do_fields, do_particles=do_particles) - test_checksum.evaluate(rtol=rtol, atol=atol) + # Reset benchmark? + reset = ( os.getenv('CHECKSUM_RESET', 'False').lower() in + ['true', '1', 't', 'y', 'yes', 'on'] ) + + if reset: + print(f"Environment variable CHECKSUM_RESET is set, resetting benchmark for {test_name}") + reset_benchmark(test_name, output_file, output_format, do_fields, do_particles) + else: + test_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) + test_checksum.evaluate(rtol=rtol, atol=atol) def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): From f044929d123e3bbe1639b28d09b31b8b8c137917 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sat, 10 Aug 2024 01:59:32 +0200 Subject: [PATCH 11/17] Make quantum_xi_c2 a non-static member variable of the WarpX class (#5073) * make quantum_xi_c2 a non-static member variable of the WarpX class * rename member variable as m_.. --- Source/FieldSolver/WarpX_QED_Field_Pushers.cpp | 2 +- Source/WarpX.H | 2 +- Source/WarpX.cpp | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index 1d3cb04a8a7..9741d9b667b 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -169,7 +169,7 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) ); // Make local copy of xi, to use on device. - const Real xi_c2 = WarpX::quantum_xi_c2; + const Real xi_c2 = m_quantum_xi_c2; // Apply QED correction to electric field, using temporary arrays. amrex::ParallelFor( diff --git a/Source/WarpX.H b/Source/WarpX.H index da60824945e..5fad8085673 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -667,7 +667,7 @@ public: */ void Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real dt); - static amrex::Real quantum_xi_c2; + amrex::Real m_quantum_xi_c2; /** Check and potentially compute load balancing */ diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4d51dd283a3..9ed59c348b3 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -100,7 +100,6 @@ bool WarpX::fft_do_time_averaging = false; amrex::IntVect WarpX::m_fill_guards_fields = amrex::IntVect(0); amrex::IntVect WarpX::m_fill_guards_current = amrex::IntVect(0); -Real WarpX::quantum_xi_c2 = PhysConst::xi_c2; Real WarpX::gamma_boost = 1._rt; Real WarpX::beta_boost = 0._rt; Vector WarpX::boost_direction = {0,0,0}; @@ -903,12 +902,15 @@ WarpX::ReadParameters () utils::parser::queryWithParser( pp_warpx, "n_current_deposition_buffer", n_current_deposition_buffer); + //Default value for the quantum parameter used in Maxwell’s QED equations + m_quantum_xi_c2 = PhysConst::xi_c2; + amrex::Real quantum_xi_tmp; const auto quantum_xi_is_specified = utils::parser::queryWithParser(pp_warpx, "quantum_xi", quantum_xi_tmp); if (quantum_xi_is_specified) { double const quantum_xi = quantum_xi_tmp; - quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); + m_quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); } const auto at_least_one_boundary_is_pml = From 75e2fd9e4ebc69f20d2dc22dfd75d38327168c29 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sat, 10 Aug 2024 02:01:26 +0200 Subject: [PATCH 12/17] Add thread sanitizer test in CI (#4799) * add thread sanitizer * move sanitizers in new yml file * fix bug * update script to try to use thread sanitizers in CI * update script to run race condition sanitizer * fix libarcher option * use docker image for newer clang in thread sanitizer test * fix bug * add new dependencies file in github workflows * add sudo installation * add sudo installation * add wget to the dependencies * add xz utils to dependencies * downgrade ubuntu version to 23.12 * correct mistake 23.12 -> 23.10 * remove wget and xz from installation list and add curl * add back wget to installation list * add back xz-utils to installation list * add git to installation list * use clang-17 * change numbers of retries * revert number of retries * use clang 17 * properly set clang 17 as the compiler to be used * add ccache to installation list * add pkg-config to install list * allow running mpi with root privilegies * fix race condition * fix bug * disable serialization of initial conditions * fix race condition * fix race conditions * fixed bug * Update .github/workflows/clang_sanitizers.yml * update AMReX * implement WeiqunZhang's suggestion to fix race conditions * increase required precision for self fields in clang sanitizers CI test * add comment line * update comment * further increase tolerance * further increase tolerance * Update .github/workflows/clang_sanitizers.yml * Apply suggestions from code review * revert change of warpx.self_fields_required_precision * add more sanitizer checks to UB sanitizer test * disable leak check * set current_correction to true when it is first declared * use double precision for thread sanitizer test * add instruction to debug CI failure * remove comment * try to debug fail in sanitizer test * fix bug in yml * add instructions to download output files in order to debug a segfault * debug issues with thread sanitizer * remove code added for debugging, compile separately for Embedded Boundary --------- Co-authored-by: Remi Lehe --- .github/workflows/clang_sanitizers.yml | 172 ++++++++++++++++++++ .github/workflows/dependencies/clang17.sh | 71 ++++++++ .github/workflows/ubuntu.yml | 55 ------- Source/Particles/ParticleBoundaryBuffer.cpp | 11 +- 4 files changed, 251 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/clang_sanitizers.yml create mode 100755 .github/workflows/dependencies/clang17.sh diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml new file mode 100644 index 00000000000..33c6646af73 --- /dev/null +++ b/.github/workflows/clang_sanitizers.yml @@ -0,0 +1,172 @@ +name: 🧴 clang sanitizers + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers + cancel-in-progress: true + +jobs: + build_UB_sanitizer: + name: Clang UB sanitizer + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang15.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-15) + export CC=$(which clang-15) + export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE \ + -DWarpX_PARTICLE_PRECISION=SINGLE + cmake --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with UB sanitizer + run: | + + export OMP_NUM_THREADS=2 + + #MPI implementations often leak memory + export "ASAN_OPTIONS=detect_leaks=0" + + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d + + build_thread_sanitizer: + name: Clang thread sanitizer + runs-on: ubuntu-22.04 + container: ubuntu:23.10 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang17.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-17) + export CC=$(which clang-17) + export CXXFLAGS="-fsanitize=thread" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=OFF \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build -j 4 + + cmake -S . -B build_EB \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="2" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=ON \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build_EB -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with thread sanitizer + run: | + export PMIX_MCA_gds=hash + export TSAN_OPTIONS='ignore_noninstrumented_modules=1' + export ARCHER_OPTIONS="verbose=1" + + # We need these two lines because these tests run inside a docker container + export OMPI_ALLOW_RUN_AS_ROOT=1 + export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + + export OMP_NUM_THREADS=2 + + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d warpx.serialize_initial_conditions = 0 + + git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data + cd Examples/Tests/embedded_circle + + ulimit -c unlimited + + mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_2d warpx.serialize_initial_conditions = 0 + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/dependencies/clang17.sh b/.github/workflows/dependencies/clang17.sh new file mode 100755 index 00000000000..d208a9f3f3b --- /dev/null +++ b/.github/workflows/dependencies/clang17.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Copyright 2024 The WarpX Community +# +# License: BSD-3-Clause-LBNL +# Authors: Luca Fedeli + +set -eu -o pipefail + +# This dependency file is currently used within a docker container, +# which does not come with sudo. +apt-get -qqq update +apt-get -y install sudo + +# `man apt.conf`: +# Number of retries to perform. If this is non-zero APT will retry +# failed files the given number of times. +echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries + +# This dependency file is currently used within a docker container, +# which does not come (among others) with wget, xz-utils, curl, git, +# ccache, and pkg-config pre-installed. +sudo apt-get -qqq update +sudo apt-get install -y \ + cmake \ + clang-17 \ + clang-tidy-17 \ + libblas-dev \ + libc++-17-dev \ + libboost-math-dev \ + libfftw3-dev \ + libfftw3-mpi-dev \ + libhdf5-openmpi-dev \ + liblapack-dev \ + libopenmpi-dev \ + libomp-17-dev \ + ninja-build \ + wget \ + xz-utils \ + curl \ + git \ + ccache \ + pkg-config + +# Use clang 17 +export CXX=$(which clang++-17) +export CC=$(which clang-17) + +# cmake-easyinstall +# +sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall +sudo chmod a+x /usr/local/bin/cmake-easyinstall +export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" + +# BLAS++ & LAPACK++ +cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/blaspp.git \ + -Duse_openmp=OFF \ + -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_VERBOSE_MAKEFILE=ON + +cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/lapackpp.git \ + -Duse_cmake_find_lapack=ON \ + -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9f4953b7a98..05e53883534 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -224,61 +224,6 @@ jobs: export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py - build_UB_sanitizer: - name: Clang UB sanitizer - runs-on: ubuntu-22.04 - if: github.event.pull_request.draft == false - env: - CC: clang - CXX: clang++ - # On CI for this test, Ninja is slower than the default: - #CMAKE_GENERATOR: Ninja - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - .github/workflows/dependencies/clang15.sh - - name: CCache Cache - uses: actions/cache@v4 - with: - path: ~/.cache/ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: build WarpX - run: | - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=10 - export CCACHE_MAXSIZE=100M - ccache -z - - export CXX=$(which clang++-15) - export CC=$(which clang-15) - export CXXFLAGS="-fsanitize=undefined -fno-sanitize-recover=all" - - cmake -S . -B build \ - -GNinja \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_FFT=ON \ - -DWarpX_QED=ON \ - -DWarpX_QED_TABLE_GEN=ON \ - -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE \ - -DWarpX_PARTICLE_PRECISION=SINGLE - cmake --build build -j 4 - - ccache -s - du -hs ~/.cache/ccache - - - name: run with UB sanitizer - run: | - export OMP_NUM_THREADS=2 - mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz - mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d - mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d - mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d - save_pr_number: if: github.event_name == 'pull_request' runs-on: ubuntu-latest diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 25068b2e65c..7b2ebce92b9 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -401,6 +401,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC for (int lev = 0; lev < pc.numLevels(); ++lev) { + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + const auto& plevel = pc.GetParticles(lev); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -408,10 +413,10 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) { continue; } - auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( - lev, pti.index(), pti.LocalTileIndex()); + auto& ptile_buffer = + species_buffer.ParticlesAt(lev, pti.index(), pti.LocalTileIndex()); + const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); if (np == 0) { continue; } From 37046f3d1c5d305ea58be2b9cd37ea68c666728e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 10 Aug 2024 21:05:10 -0500 Subject: [PATCH 13/17] Release 24.08 (#5125) * AMReX: 24.08 * pyAMReX: 24.08 * WarpX: 24.08 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 8f5e59ccaa4..22c09561062 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - + cd ../amrex && git checkout --detach 24.08 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c81a057131..f74820b3015 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.07) +project(WarpX VERSION 24.08) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 7997b1e338c..a1ef15b09bf 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.07' +version = u'24.08' # The full version, including alpha/beta/rc tags. -release = u'24.07' +release = u'24.08' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 31f35eeceac..ed0d4ce29e0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.07', + version = '24.08', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 68d59d850a0..7d111741389 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 +branch = 24.08 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 96dbea5c380..549b671138c 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 +branch = 24.08 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index f1e5b3f62e2..39d88010517 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.07 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.08 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "20e6f2eadf0c297517588ba38973ec7c7084fa31" +set(WarpX_amrex_branch "24.08" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 133747aaeae..020502d9921 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.07 CONFIG REQUIRED) + find_package(pyAMReX 24.08 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "e007e730d48cb5fdbe1e10462d7d0a14e2bc8f8e" +set(WarpX_pyamrex_branch "24.08" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 275b7b5efb0..425841d1970 100755 --- a/run_test.sh +++ b/run_test.sh @@ -72,7 +72,7 @@ python3 -m pip cache purge # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - +cd amrex && git checkout --detach 24.08 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index f05d5a71899..30f91d8fb4a 100644 --- a/setup.py +++ b/setup.py @@ -280,7 +280,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.07', + version = '24.08', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From e39e3c5e7a739778e239a5681e996ec5515f4b47 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 12 Aug 2024 11:39:26 -0500 Subject: [PATCH 14/17] CI: heFFTe for tests (#5121) * CI: heFFTe for tests Add heFFTe to Azure runners to enable runtime tests that need it. * CUDA & HIP: heFFTe Builds --- .azure-pipelines.yml | 11 +++++++++++ .github/workflows/cuda.yml | 16 ++++++++++++++++ .github/workflows/dependencies/hip.sh | 20 ++++++++++++++++++++ .github/workflows/hip.yml | 6 ++++-- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7c71599bbf3..fa14902283c 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -110,6 +110,17 @@ jobs: -DCMAKE_CXX_STANDARD=17 \ -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON fi + if [[ "${WARPX_CI_REGULAR_CARTESIAN_3D:-FALSE}" == "TRUE" ]]; then + cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON + fi rm -rf ${CEI_TMP} df -h displayName: 'Install dependencies' diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 22c09561062..95d9c148b97 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -43,6 +43,11 @@ jobs: export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} + which nvcc || echo "nvcc not in PATH!" + cmake-easyinstall --prefix=/usr/local \ git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ -DopenPMD_USE_PYTHON=OFF \ @@ -51,6 +56,16 @@ jobs: -DBUILD_CLI_TOOLS=OFF \ -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON + cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | export CCACHE_COMPRESS=1 @@ -71,6 +86,7 @@ jobs: -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index ae897786acd..2a1b4d090bc 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -65,6 +65,13 @@ source /etc/profile.d/rocm.sh hipcc --version which clang which clang++ +export CXX=$(which clang++) +export CC=$(which clang) + +# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3 +# https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786 +# https://github.com/open-mpi/ompi/issues/9317 +export LDFLAGS="-lopen-pal" # cmake-easyinstall # @@ -72,3 +79,16 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + +# heFFTe +# +cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0f6710b1405..8b89e3dc4d0 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -55,7 +55,8 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_FFT=ON + -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -115,7 +116,8 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_FFT=ON + -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF From d400c7bcd83fe355fbaac69807ddfd5807f6c2f2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 12 Aug 2024 12:22:47 -0500 Subject: [PATCH 15/17] CMake: More Pip Control Options (#5128) - `PY_PIP_OPTIONS`: Passed to `python -m pip ...` - `PY_PIP_INSTALL_OPTIONS`: Passed to `python -m pip install ...` (formerly `PYINSTALLOPTIONS`) --- CMakeLists.txt | 12 +++++++----- Docs/source/install/cmake.rst | 6 ++++-- Regression/WarpX-tests.ini | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f74820b3015..4219b33f78b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -711,7 +711,9 @@ endforeach() # pip helpers for the pywarpx package ######################################### # if(WarpX_PYTHON) - set(PYINSTALLOPTIONS "" CACHE STRING + set(PY_PIP_OPTIONS "-v" CACHE STRING + "Additional parameters to pass to `pip`") + set(PY_PIP_INSTALL_OPTIONS "" CACHE STRING "Additional parameters to pass to `pip install`") # ensure all targets are built before we package them in a wheel @@ -735,7 +737,7 @@ if(WarpX_PYTHON) ${CMAKE_COMMAND} -E rm -f -r warpx-whl COMMAND ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$ - ${Python_EXECUTABLE} -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} wheel --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -749,7 +751,7 @@ if(WarpX_PYTHON) set(pyWarpX_REQUIREMENT_FILE "requirements.txt") endif() add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements - ${Python_EXECUTABLE} -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install ${PY_PIP_INSTALL_OPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" WORKING_DIRECTORY ${WarpX_BINARY_DIR} ) @@ -766,7 +768,7 @@ if(WarpX_PYTHON) # because otherwise pip would also force reinstall all dependencies. add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -779,7 +781,7 @@ if(WarpX_PYTHON) # this is for package managers only add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index fde927c10e1..79f49b5aad1 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -83,7 +83,6 @@ CMake Option Default & Values Descr ``CMAKE_BUILD_TYPE`` RelWithDebInfo/**Release**/Debug `Type of build, symbols & optimizations `__ ``CMAKE_INSTALL_PREFIX`` system-dependent path `Install path prefix `__ ``CMAKE_VERBOSE_MAKEFILE`` ON/**OFF** `Print all compiler commands to the terminal during build `__ -``PYINSTALLOPTIONS`` Additional options for ``pip install``, e.g., ``-v --user`` ``WarpX_APP`` **ON**/OFF Build the WarpX executable application ``WarpX_ASCENT`` ON/**OFF** Ascent in situ visualization ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend @@ -104,6 +103,9 @@ CMake Option Default & Values Descr ``WarpX_QED_TOOLS`` ON/**OFF** Build external tool to generate QED lookup tables (requires PICSAR and Boost) ``WarpX_QED_TABLES_GEN_OMP`` **AUTO**/ON/OFF Enables OpenMP support for QED lookup tables generation ``WarpX_SENSEI`` ON/**OFF** SENSEI in situ visualization +``Python_EXECUTABLE`` (newest found) Path to Python executable +``PY_PIP_OPTIONS`` ``-v`` Additional options for ``pip``, e.g., ``-vvv`` +``PY_PIP_INSTALL_OPTIONS`` Additional options for ``pip install``, e.g., ``--user`` ============================= ============================================ ========================================================= WarpX can be configured in further detail with options from AMReX, which are documented in the AMReX manual: @@ -239,7 +241,7 @@ Developers could now change the WarpX source code and then call the build line a .. tip:: If you do *not* develop with :ref:`a user-level package manager `, e.g., because you rely on a HPC system's environment modules, then consider to set up a virtual environment via `Python venv `__. - Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPYINSTALLOPTIONS="--user"``. + Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPY_PIP_INSTALL_OPTIONS="--user"``. .. _building-pip-python: diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 549b671138c..dcf53b5cc98 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -65,7 +65,7 @@ branch = 24.08 dir = /home/regtester/AMReX_RegTesting/warpx branch = development cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_PYTHON_IPO=OFF -DpyAMReX_IPO=OFF -# -DPYINSTALLOPTIONS="--disable-pip-version-check" +# -DPY_PIP_INSTALL_OPTIONS="--disable-pip-version-check" # individual problems follow From f605690446b402bea50eeef75f081ab9308ddbe8 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 12 Aug 2024 23:08:01 +0200 Subject: [PATCH 16/17] Move PerformanceHints from WarpX class into anonymous namespace in WarpXInitData.cpp (#4893) * move PerformanceHints from WarpX class into anonymous namespace in WarpXInitData.cpp * fix clang-tidy issue --- Source/Initialization/WarpXInitData.cpp | 141 ++++++++++++------------ Source/WarpX.H | 3 - 2 files changed, 73 insertions(+), 71 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 1d5e859c227..eb93d121313 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -106,6 +106,71 @@ namespace WARPX_ALWAYS_ASSERT_WITH_MESSAGE(vc.allGT(gc), ss_msg.str()); } } + + /** + * \brief Check the requested resources and write performance hints + * + * @param[in] total_nboxes total number of boxes in the simulation + * @param[in] nprocs number of MPI processes + */ + void PerformanceHints (const amrex::Long total_nboxes, const amrex::Long nprocs) + { + // Check: are there more MPI ranks than Boxes? + if (nprocs > total_nboxes) { + std::stringstream warnMsg; + warnMsg << "Too many resources / too little work!\n" + << " It looks like you requested more compute resources than " + << "there are total number of boxes of cells available (" + << total_nboxes << "). " + << "You started with (" << nprocs + << ") MPI ranks, so (" << nprocs - total_nboxes + << ") rank(s) will have no work.\n" + #ifdef AMREX_USE_GPU + << " On GPUs, consider using 1-8 boxes per GPU that together fill " + << "each GPU's memory sufficiently. If you do not rely on dynamic " + << "load-balancing, then one large box per GPU is ideal.\n" + #endif + << "Consider decreasing the amr.blocking_factor and " + << "amr.max_grid_size parameters and/or using fewer MPI ranks.\n" + << " More information:\n" + << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; + + ablastr::warn_manager::WMRecordWarning( + "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + } + + #ifdef AMREX_USE_GPU + // Check: Are there more than 12 boxes per GPU? + if (total_nboxes > nprocs * 12) { + std::stringstream warnMsg; + warnMsg << "Too many boxes per GPU!\n" + << " It looks like you split your simulation domain " + << "in too many boxes (" << total_nboxes << "), which " + << "results in an average number of (" + << amrex::Long(total_nboxes/nprocs) << ") per GPU. " + << "This causes severe overhead in the communication of " + << "border/guard regions.\n" + << " On GPUs, consider using 1-8 boxes per GPU that together fill " + << "each GPU's memory sufficiently. If you do not rely on dynamic " + << "load-balancing, then one large box per GPU is ideal.\n" + << "Consider increasing the amr.blocking_factor and " + << "amr.max_grid_size parameters and/or using more MPI ranks.\n" + << " More information:\n" + << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; + + ablastr::warn_manager::WMRecordWarning( + "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + } + #endif + + // TODO: warn if some ranks have disproportionally more work than all others + // tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down + // all other ranks; we need to measure this with our load-balancing + // routines and issue a warning only of some ranks stall all other ranks + // TODO: check MPI-rank to GPU ratio (should be 1:1) + // TODO: check memory per MPI rank, especially if GPUs are underutilized + // TODO: CPU tiling hints with OpenMP + } } void @@ -527,7 +592,14 @@ WarpX::InitData () } } - PerformanceHints(); + // Computes available boxes on all levels. + amrex::Long total_nboxes = 0; + for (int ilev = 0; ilev <= finestLevel(); ++ilev) { + total_nboxes += boxArray(ilev).size(); + } + auto const nprocs = ParallelDescriptor::NProcs(); + + ::PerformanceHints(total_nboxes, nprocs); CheckKnownIssues(); } @@ -1115,73 +1187,6 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( } } -void -WarpX::PerformanceHints () -{ - // Check requested MPI ranks and available boxes - amrex::Long total_nboxes = 0; // on all MPI ranks - for (int ilev = 0; ilev <= finestLevel(); ++ilev) { - total_nboxes += boxArray(ilev).size(); - } - auto const nprocs = ParallelDescriptor::NProcs(); - - // Check: are there more MPI ranks than Boxes? - if (nprocs > total_nboxes) { - std::stringstream warnMsg; - warnMsg << "Too many resources / too little work!\n" - << " It looks like you requested more compute resources than " - << "there are total number of boxes of cells available (" - << total_nboxes << "). " - << "You started with (" << nprocs - << ") MPI ranks, so (" << nprocs - total_nboxes - << ") rank(s) will have no work.\n" -#ifdef AMREX_USE_GPU - << " On GPUs, consider using 1-8 boxes per GPU that together fill " - << "each GPU's memory sufficiently. If you do not rely on dynamic " - << "load-balancing, then one large box per GPU is ideal.\n" -#endif - << "Consider decreasing the amr.blocking_factor and " - << "amr.max_grid_size parameters and/or using fewer MPI ranks.\n" - << " More information:\n" - << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; - - ablastr::warn_manager::WMRecordWarning( - "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - } - -#ifdef AMREX_USE_GPU - // Check: Are there more than 12 boxes per GPU? - if (total_nboxes > nprocs * 12) { - std::stringstream warnMsg; - warnMsg << "Too many boxes per GPU!\n" - << " It looks like you split your simulation domain " - << "in too many boxes (" << total_nboxes << "), which " - << "results in an average number of (" - << amrex::Long(total_nboxes/nprocs) << ") per GPU. " - << "This causes severe overhead in the communication of " - << "border/guard regions.\n" - << " On GPUs, consider using 1-8 boxes per GPU that together fill " - << "each GPU's memory sufficiently. If you do not rely on dynamic " - << "load-balancing, then one large box per GPU is ideal.\n" - << "Consider increasing the amr.blocking_factor and " - << "amr.max_grid_size parameters and/or using more MPI ranks.\n" - << " More information:\n" - << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; - - ablastr::warn_manager::WMRecordWarning( - "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - } -#endif - - // TODO: warn if some ranks have disproportionally more work than all others - // tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down - // all other ranks; we need to measure this with our load-balancing - // routines and issue a warning only of some ranks stall all other ranks - // TODO: check MPI-rank to GPU ratio (should be 1:1) - // TODO: check memory per MPI rank, especially if GPUs are underutilized - // TODO: CPU tiling hints with OpenMP -} - void WarpX::CheckGuardCells() { for (int lev = 0; lev <= finest_level; ++lev) diff --git a/Source/WarpX.H b/Source/WarpX.H index 5fad8085673..2caea95e926 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1399,9 +1399,6 @@ private: */ void CheckKnownIssues(); - /** Check the requested resources and write performance hints */ - void PerformanceHints (); - void BuildBufferMasks (); [[nodiscard]] const amrex::iMultiFab* getCurrentBufferMasks (int lev) const { From 502e5ea31c1334f0083bed64eb8f8202b1100e81 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:17:39 -0700 Subject: [PATCH 17/17] CI: update option name to clean test dirs (#5129) --- run_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_test.sh b/run_test.sh index 425841d1970..11360673b63 100755 --- a/run_test.sh +++ b/run_test.sh @@ -83,7 +83,7 @@ curl -sOL https://github.com/openPMD/openPMD-example-datasets/raw/4ba1d257c5b489 cd - # Clone the AMReX regression test utility -git clone -b EZoni_rm_testdir https://github.com/EZoni/regression_testing.git +git clone https://github.com/AMReX-Codes/regression_testing.git # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks @@ -99,7 +99,7 @@ echo "cd $PWD" if [ -z "${WARPX_CI_CLEAN_TESTS}" ]; then test_rm_dir="" else - test_rm_dir="--rm_testdir" + test_rm_dir="--clean_testdir" fi # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then