Merge branch 'ECP-WarpX:development' into add_feature_projection_div_…

…cleaner

.github/workflows/cuda.yml

@@ -131,7 +131,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 24.08 && cd -
+        cd ../amrex && git checkout --detach d9919c92db09ed9e927f9e4c7fb2ddcc161ab5ed && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

Examples/Tests/collision/analysis_collision_1d.py

@@ -123,4 +123,4 @@
 assert error < tolerance
 
 test_name = os.path.split(os.getcwd())[1]
-checksumAPI.evaluate_checksum(test_name, fn)
+checksumAPI.evaluate_checksum(test_name, fn, rtol=2.5e-3)

Regression/WarpX-GPU-tests.ini

@@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more
 
 [AMReX]
 dir = /home/regtester/git/amrex/
-branch = 24.08
+branch = d9919c92db09ed9e927f9e4c7fb2ddcc161ab5ed
 
 [source]
 dir = /home/regtester/git/WarpX

Regression/WarpX-tests.ini

@@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det
 
 [AMReX]
 dir = /home/regtester/AMReX_RegTesting/amrex/
-branch = 24.08
+branch = d9919c92db09ed9e927f9e4c7fb2ddcc161ab5ed
 
 [source]
 dir = /home/regtester/AMReX_RegTesting/warpx
@@ -3856,7 +3856,7 @@ cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 outputFile = Point_of_contact_EB_3d_plt
 analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py
@@ -3871,22 +3871,22 @@ cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 outputFile = Point_of_contact_EB_rz_plt
 analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py
 
 [ThetaImplicitPicard_1d]
 buildDir = .
 inputFile = Examples/Tests/Implicit/inputs_1d
-runtime_params = warpx.abort_on_warning_threshold=high
+runtime_params = warpx.abort_on_warning_threshold=high amr.max_grid_size=32
 dim = 1
 addToCompileString =
 cmakeSetupOpts = -DWarpX_DIMS=1
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py
 
@@ -3900,7 +3900,7 @@ cmakeSetupOpts = -DWarpX_DIMS=2
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py
 
@@ -3916,21 +3916,21 @@ target = pip_install
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py
 
 [SemiImplicitPicard_1d]
 buildDir = .
 inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit
-runtime_params = warpx.abort_on_warning_threshold=high
+runtime_params = warpx.abort_on_warning_threshold=high amr.max_grid_size=32
 dim = 1
 addToCompileString =
 cmakeSetupOpts = -DWarpX_DIMS=1
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py
 
@@ -3943,7 +3943,7 @@ cmakeSetupOpts = -DWarpX_DIMS=2
 restartTest = 0
 useMPI = 1
 numprocs = 2
-useOMP = 0
+useOMP = 1
 numthreads = 1
 analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py
 

Source/Parallelization/GuardCellManager.cpp

@@ -101,9 +101,9 @@ guardCellManager::Init (
     // the fine grid by a number of cells equal to the ref_ratio in the moving
     // window direction.
     if (do_moving_window) {
-        WARPX_ALWAYS_ASSERT_WITH_MESSAGE(ref_ratios.size() <= 1,
+        WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level <= 1,
             "The number of grow cells for the moving window currently assumes 2 levels max.");
-        const auto nlevs = static_cast<int>(ref_ratios.size()+1);
+        const auto nlevs = static_cast<int>(max_level+1);
         const int max_r = (nlevs > 1) ? ref_ratios[0][moving_window_dir] : 2;
 
         ngx = std::max(ngx,max_r);

Source/Particles/Resampling/VelocityCoincidenceThinning.H

@@ -123,7 +123,7 @@ public:
         void labelOnSphericalVelocityGrid (const amrex::ParticleReal ux[],
                                            const amrex::ParticleReal uy[],
                                            const amrex::ParticleReal uz[],
-                                           const unsigned int indices[],
+                                           const int indices[],
                                            int bin_array[], int index_array[],
                                            const int cell_start, const int cell_stop ) const
         {
@@ -151,7 +151,7 @@ public:
         void labelOnCartesianVelocityGrid (const amrex::ParticleReal ux[],
                                            const amrex::ParticleReal uy[],
                                            const amrex::ParticleReal uz[],
-                                           const unsigned int indices[],
+                                           const int indices[],
                                            int bin_array[], int index_array[],
                                            const int cell_start, const int cell_stop ) const
         {
@@ -168,7 +168,7 @@ public:
 
         AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
         void operator() (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[],
-                         const amrex::ParticleReal uz[], const unsigned int indices[],
+                         const amrex::ParticleReal uz[], const int indices[],
                          int bin_array[], int index_array[],
                          const int cell_start, const int cell_stop) const
         {

Source/ablastr/math/fft/AnyFFT.H

@@ -15,12 +15,14 @@
 
 #   if defined(AMREX_USE_CUDA)
 #       include <cufft.h>
+#       include <cuComplex.h>
 #   elif defined(AMREX_USE_HIP)
 #       if __has_include(<rocfft/rocfft.h>)  // ROCm 5.3+
 #           include <rocfft/rocfft.h>
 #       else
 #           include <rocfft.h>
 #       endif
+#       include <hip/hip_complex.h>
 #   elif defined(AMREX_USE_SYCL)
 #       include <oneapi/mkl/dfti.hpp>
 #   else
@@ -75,6 +77,28 @@ namespace ablastr::math::anyfft
 #       endif
 #   endif
 
+    /* Library-dependent multiply helpers */
+#   if defined(AMREX_USE_CUDA)
+#       ifdef AMREX_USE_FLOAT
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = cuCmulf(a, b); }
+#       else
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = cuCmul(a, b); }
+#       endif
+#   elif defined(AMREX_USE_HIP)
+    #   ifdef AMREX_USE_FLOAT
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = hipCmulf(a, b); }
+#       else
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = hipCmul(a, b); }
+#       endif
+#   elif defined(AMREX_USE_SYCL)
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = a * b; }
+#   else
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) {
+        c[0] = a[0] * b[0] - a[1] * b[1];
+        c[1] = a[0] * b[1] + a[1] * b[0];
+    }
+#   endif
+
     /** Library-dependent FFT plans type, which holds one fft plan per box
      * (plans are only initialized for the boxes that are owned by the local MPI rank).
      */

Source/ablastr/math/fft/WrapCuFFT.cpp

@@ -56,7 +56,6 @@ namespace ablastr::math::anyfft
                 result = cufftPlan2d(
                     &(fft_plan.m_plan), real_size[1], real_size[0], VendorC2R);
             } else if (dim == 1) {
-                int batch = 2;
                 result = cufftPlan1d(
                     &(fft_plan.m_plan), real_size[0], VendorC2R, 1);
             } else {

cmake/dependencies/AMReX.cmake

@@ -273,7 +273,7 @@ set(WarpX_amrex_src ""
 set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git"
     CACHE STRING
     "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)")
-set(WarpX_amrex_branch "24.08"
+set(WarpX_amrex_branch "d9919c92db09ed9e927f9e4c7fb2ddcc161ab5ed"
     CACHE STRING
     "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)")
 

cmake/dependencies/pyAMReX.cmake

@@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON)
 set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git"
     CACHE STRING
     "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)")
-set(WarpX_pyamrex_branch "24.08"
+set(WarpX_pyamrex_branch "abdf332e25bfeef2b4d613d7adbe93fb8cf3e2f7"
     CACHE STRING
     "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)")
 

run_test.sh

@@ -72,7 +72,7 @@ python3 -m pip cache purge
 
 # Clone AMReX and warpx-data
 git clone https://github.com/AMReX-Codes/amrex.git
-cd amrex && git checkout --detach 24.08 && cd -
+cd amrex && git checkout --detach d9919c92db09ed9e927f9e4c7fb2ddcc161ab5ed && cd -
 # warpx-data contains various required data sets
 git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git
 # openPMD-example-datasets contains various required data sets