Merge branch 'development' into plasma_chamber

.github/workflows/cuda.yml

@@ -126,7 +126,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 96db0a665ff1e6bbe638490fd02d3aafb9188f6b && cd -
+        cd ../amrex && git checkout --detach b3f67385e62f387b548389222840486c0fffca57 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

.github/workflows/dependencies/hip.sh

@@ -53,6 +53,7 @@ sudo apt-get install -y --no-install-recommends \
     rocm-dev        \
     rocfft-dev      \
     rocprim-dev     \
+    rocsparse-dev   \
     rocrand-dev     \
     hiprand-dev
 

.github/workflows/dependencies/nvcc11-3.sh

@@ -41,7 +41,8 @@ sudo apt-get install -y          \
     cuda-nvml-dev-11-3           \
     cuda-nvtx-11-3               \
     libcufft-dev-11-3            \
-    libcurand-dev-11-3
+    libcurand-dev-11-3           \
+    libcusparse-dev-11-3
 sudo ln -s cuda-11.3 /usr/local/cuda
 
 # if we run out of temporary storage in CI:

.github/workflows/dependencies/nvcc11-8.sh

@@ -41,7 +41,8 @@ sudo apt-get install -y          \
     cuda-nvml-dev-11-8           \
     cuda-nvtx-11-8               \
     libcufft-dev-11-8            \
-    libcurand-dev-11-8
+    libcurand-dev-11-8           \
+    libcusparse-dev-11-8
 sudo ln -s cuda-11.8 /usr/local/cuda
 
 # if we run out of temporary storage in CI:

.pre-commit-config.yaml

@@ -69,7 +69,7 @@ repos:
 # Python: Ruff linter & formatter
 #   https://docs.astral.sh/ruff/
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.2
+  rev: v0.8.3
   hooks:
     # Run the linter
     - id: ruff

Docs/source/developers/testing.rst

@@ -140,8 +140,8 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            1  # dims
            2  # nprocs
            inputs_test_1d_laser_acceleration  # inputs
-           analysis.py  # analysis
-           diags/diag1000100  # output (plotfile)
+           "analysis.py diags/diag1000100"  # analysis
+           "analysis_default_regression.py --path diags/diag1000100"  # checksum
            OFF  # dependency
        )
 
@@ -154,8 +154,8 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            2  # dims
            2  # nprocs
            inputs_test_2d_laser_acceleration_picmi.py  # inputs
-           analysis.py  # analysis
-           diags/diag1000100  # output (plotfile)
+           "analysis.py diags/diag1000100"  # analysis
+           "analysis_default_regression.py --path diags/diag1000100"  # checksum
            OFF  # dependency
        )
 
@@ -168,14 +168,14 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            3  # dims
            2  # nprocs
            inputs_test_3d_laser_acceleration_restart  # inputs
-           analysis_default_restart.py  # analysis
-           diags/diag1000100  # output (plotfile)
+           "analysis_default_restart.py diags/diag1000100"  # analysis
+           "analysis_default_regression.py --path diags/diag1000100"  # checksum
            test_3d_laser_acceleration  # dependency
        )
 
   Note that the restart has an explicit dependency, namely it can run only provided that the original test, from which the restart checkpoint files will be read, runs first.
 
-* A more complex example. Add the **PICMI test** ``test_rz_laser_acceleration_picmi``, with custom command-line arguments ``--test`` and ``dir``, and openPMD time series output:
+* A more complex example. Add the **PICMI test** ``test_rz_laser_acceleration_picmi``, with custom command-line arguments ``--test`` and ``dir``, openPMD time series output, and custom command line arguments for the checksum comparison:
 
   .. code-block:: cmake
 
@@ -184,18 +184,38 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            RZ  # dims
            2   # nprocs
            "inputs_test_rz_laser_acceleration_picmi.py --test --dir 1"  # inputs
-           analysis.py  # analysis
-           diags/diag1/  # output (openPMD time series)
+           "analysis.py diags/diag1/"  # analysis
+           "analysis_default_regression.py --path diags/diag1/ --skip-particles --rtol 1e-7"  # checksum
            OFF  # dependency
        )
 
+The ``analysis`` and ``checksum`` commands passed as arguments to ``add_warpx_test`` can be set to ``OFF`` if the intention is to skip the respective analysis for a given test.
+
 If you need a new Python package dependency for testing, please add it in `Regression/requirements.txt <https://github.com/ECP-WarpX/WarpX/blob/development/Regression/requirements.txt>`__.
 
 Sometimes two or more tests share a large number of input parameters.
 The shared input parameters can be collected in a "base" input file that can be passed as a runtime parameter in the actual test input files through the parameter ``FILE``.
 
 If the new test is added in a new directory that did not exist before, please add the name of that directory with the command ``add_subdirectory`` in `Physics_applications/CMakeLists.txt <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Physics_applications/CMakeLists.txt>`__ or `Tests/CMakeLists.txt <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Tests/CMakeLists.txt>`__, depending on where the new test directory is located.
 
+If not already present, the default regression analysis script ``analysis_default_regression.py`` in the examples above must be linked from `Examples/analysis_default_regression.py <https://github.com/ECP-WarpX/WarpX/blob/development/Examples/analysis_default_regression.py>`__, by executing once the following command from the test directory:
+
+  .. code-block:: bash
+
+       ln -s ../../analysis_default_regression.py analysis_default_regression.py
+
+Here is the help message of the default regression analysis script, including usage and list of available options and arguments:
+
+  .. code-block:: bash
+
+       usage: analysis_default_regression.py [-h] [--path PATH] [--rtol RTOL] [--skip-fields] [--skip-particles]
+       options:
+         -h, --help        show this help message and exit
+         --path PATH       path to output file(s)
+         --rtol RTOL       relative tolerance to compare checksums
+         --skip-fields     skip fields when comparing checksums
+         --skip-particles  skip particles when comparing checksums
+
 Naming conventions for automated tests
 --------------------------------------
 

Docs/source/install/hpc/dane.rst

@@ -24,18 +24,19 @@ If you are new to this system, **please see the following resources**:
 Preparation
 -----------
 
-Use the following commands to download the WarpX source code:
+Use the following commands to download the WarpX source code.
+Note that these commands and the shell scripts all assume the bash shell.
 
 .. code-block:: bash
 
-   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+   git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/dane/src/warpx
 
 We use system software modules, add environment hints and further dependencies via the file ``$HOME/dane_warpx.profile``.
 Create it now:
 
 .. code-block:: bash
 
-   cp $HOME/src/warpx/Tools/machines/dane-llnl/dane_warpx.profile.example $HOME/dane_warpx.profile
+   cp /usr/workspace/${USER}/dane/src/warpx/Tools/machines/dane-llnl/dane_warpx.profile.example $HOME/dane_warpx.profile
 
 .. dropdown:: Script Details
    :color: light
@@ -67,7 +68,7 @@ Finally, since Dane does not yet provide software modules for some of our depend
 
 .. code-block:: bash
 
-   bash $HOME/src/warpx/Tools/machines/dane-llnl/install_dependencies.sh
+   bash /usr/workspace/${USER}/dane/src/warpx/Tools/machines/dane-llnl/install_dependencies.sh
    source /usr/workspace/${USER}/dane/venvs/warpx-dane/bin/activate
 
 .. dropdown:: Script Details
@@ -88,13 +89,13 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd /usr/workspace/${USER}/dane/src/warpx
    rm -rf build_dane
 
    cmake -S . -B build_dane -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
    cmake --build build_dane -j 6
 
-The WarpX application executables are now in ``$HOME/src/warpx/build_dane/bin/``.
+The WarpX application executables are now in ``/usr/workspace/${USER}/dane/src/warpx/build_dane/bin/``.
 Additionally, the following commands will install WarpX as a Python module:
 
 .. code-block:: bash
@@ -118,7 +119,7 @@ If you already installed WarpX in the past and want to update it, start by getti
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd /usr/workspace/${USER}/dane/src/warpx
 
    # read the output of this command - does it look ok?
    git status
@@ -137,7 +138,7 @@ And, if needed,
 - log out and into the system, activate the now updated environment profile as usual,
 - :ref:`execute the dependency install scripts <building-dane-preparation>`.
 
-As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_dane`` and rebuild WarpX.
+As a last step, clean the build directory ``rm -rf /usr/workspace/${USER}/dane/src/warpx/build_dane`` and rebuild WarpX.
 
 
 .. _running-cpp-dane:

Docs/source/usage/parameters.rst

@@ -290,6 +290,21 @@ Overall simulation parameters
         In electromagnetic mode, this solver can be used to initialize the species' self fields
         (``<species_name>.initialize_self_fields=1``) provided that the field BCs are PML (``boundary.field_lo,hi = PML``).
 
+          * ``warpx.use_2d_slices_fft_solver`` (`bool`) optional (default: 0): Select the type of Integrated Green Function solver.
+            If 0, solve Poisson equation in full 3D geometry.
+            If 1, solve Poisson equation in a quasi 3D geometry, neglecting the :math:`z` derivatives in the Laplacian of the Poisson equation.
+            In practice, in this case, the code performes many 2D Poisson solves on all :math:`(x,y)` slices, each slice at a given :math:`z`.
+            This is often a good approximation for ultra-relativistic beams propagating along the :math:`z` direction, with the relativistic solver.
+            As a consequence, this solver does not need to do an FFT along the :math:`z` direction,
+            and instead uses only transverse FFTs (along :math:`x` and :math:`y`) at each :math:`z` position (or :math:`z` "slice").
+
+          * ``ablastr.nprocs_igf_fft`` (`int`) optional (default: number of MPI ranks): Number of MPI ranks used to parallalelize the FFT solver.
+            This can be less or equal than then number of MPI ranks that are used to run the overall simulation.
+            It can be useful if the auxiliary simulation boxes fit within a single process, so to avoid extra communications.
+            The auxiliary boxes are extended boxes in real and spectral space that are used to perform the necessary FFTs.
+            The extended simulation box size in real space is :math:`2n_x-1, 2n_y-1, 2n_z-1` with the 3D solver, :math:`2n_x-1, 2n_y -1, n_z` with the 2D solver.
+            The extended simulation box size in spectral space is :math:`n_x, 2n_y-1, 2n_z-1` with the 3D solver, :math:`n_x, 2n_y-1, n_z` with the 2D solver.
+
 * ``warpx.self_fields_required_precision`` (`float`, default: 1.e-11)
     The relative precision with which the electrostatic space-charge fields should
     be calculated. More specifically, the space-charge fields are

Examples/CMakeLists.txt

@@ -21,8 +21,8 @@ endif()
 # dims: 1,2,RZ,3
 # nprocs: 1 or 2 (maybe refactor later on to just depend on WarpX_MPI)
 # inputs: inputs file or PICMI script, WarpX_MPI decides w/ or w/o MPI
-# analysis: analysis script, always run without MPI
-# output: output file(s) to analyze
+# analysis: custom test analysis command, always run without MPI
+# checksum: default regression analysis command (checksum benchmark)
 # dependency: name of base test that must run first
 #
 function(add_warpx_test
@@ -31,7 +31,7 @@ function(add_warpx_test
     nprocs
     inputs
     analysis
-    output
+    checksum
     dependency
 )
     # cannot run MPI tests w/o MPI build
@@ -72,14 +72,25 @@ function(add_warpx_test
     separate_arguments(ANALYSIS_LIST UNIX_COMMAND "${analysis}")
     list(GET ANALYSIS_LIST 0 ANALYSIS_FILE)
     cmake_path(SET ANALYSIS_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${ANALYSIS_FILE}")
-    # TODO Enable lines below to handle command-line arguments
-    #list(LENGTH ANALYSIS_LIST ANALYSIS_LIST_LENGTH)
-    #if(ANALYSIS_LIST_LENGTH GREATER 1)
-    #    list(SUBLIST ANALYSIS_LIST 1 -1 ANALYSIS_ARGS)
-    #    list(JOIN ANALYSIS_ARGS " " ANALYSIS_ARGS)
-    #else()
-    #    set(ANALYSIS_ARGS "")
-    #endif()
+    list(LENGTH ANALYSIS_LIST ANALYSIS_LIST_LENGTH)
+    if(ANALYSIS_LIST_LENGTH GREATER 1)
+        list(SUBLIST ANALYSIS_LIST 1 -1 ANALYSIS_ARGS)
+        list(JOIN ANALYSIS_ARGS " " ANALYSIS_ARGS)
+    else()
+        set(ANALYSIS_ARGS "")
+    endif()
+
+    # get checksum script and optional command-line arguments
+    separate_arguments(CHECKSUM_LIST UNIX_COMMAND "${checksum}")
+    list(GET CHECKSUM_LIST 0 CHECKSUM_FILE)
+    cmake_path(SET CHECKSUM_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${CHECKSUM_FILE}")
+    list(LENGTH CHECKSUM_LIST CHECKSUM_LIST_LENGTH)
+    if(CHECKSUM_LIST_LENGTH GREATER 1)
+        list(SUBLIST CHECKSUM_LIST 1 -1 CHECKSUM_ARGS)
+        list(JOIN CHECKSUM_ARGS " " CHECKSUM_ARGS)
+    else()
+        set(CHECKSUM_ARGS "")
+    endif()
 
     # Python test?
     set(python OFF)
@@ -175,25 +186,52 @@ function(add_warpx_test
 
     # test analysis
     if(analysis)
+        # for argparse, do not pass command-line arguments as one quoted string
+        separate_arguments(ANALYSIS_ARGS UNIX_COMMAND "${ANALYSIS_ARGS}")
         add_test(
             NAME ${name}.analysis
             COMMAND
-                ${THIS_Python_SCRIPT_EXE} ${ANALYSIS_FILE}
-                ${output}
+                ${THIS_Python_SCRIPT_EXE}
+                ${ANALYSIS_FILE}
+                ${ANALYSIS_ARGS}
             WORKING_DIRECTORY ${THIS_WORKING_DIR}
         )
         # test analysis depends on test run
         set_property(TEST ${name}.analysis APPEND PROPERTY DEPENDS "${name}.run")
         # FIXME Use helper function to handle Windows exceptions
         set(PYTHONPATH "$ENV{PYTHONPATH}:${CMAKE_PYTHON_OUTPUT_DIRECTORY}")
         # add paths for custom Python modules
-        set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Regression/Checksum")
         set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Regression/PostProcessingUtils")
         set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Tools/Parser")
         set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Tools/PostProcessing")
         set_property(TEST ${name}.analysis APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PYTHONPATH}")
     endif()
 
+    # checksum analysis
+    if(checksum)
+        # for argparse, do not pass command-line arguments as one quoted string
+        separate_arguments(CHECKSUM_ARGS UNIX_COMMAND "${CHECKSUM_ARGS}")
+        add_test(
+            NAME ${name}.checksum
+            COMMAND
+                ${THIS_Python_SCRIPT_EXE}
+                ${CHECKSUM_FILE}
+                ${CHECKSUM_ARGS}
+            WORKING_DIRECTORY ${THIS_WORKING_DIR}
+        )
+        # test analysis depends on test run
+        set_property(TEST ${name}.checksum APPEND PROPERTY DEPENDS "${name}.run")
+        if(analysis)
+            # checksum analysis depends on test analysis
+            set_property(TEST ${name}.checksum APPEND PROPERTY DEPENDS "${name}.analysis")
+        endif()
+        # FIXME Use helper function to handle Windows exceptions
+        set(PYTHONPATH "$ENV{PYTHONPATH}:${CMAKE_PYTHON_OUTPUT_DIRECTORY}")
+        # add paths for custom Python modules
+        set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Regression/Checksum")
+        set_property(TEST ${name}.checksum APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PYTHONPATH}")
+    endif()
+
     # CI: remove test directory after run
     if(WarpX_TEST_CLEANUP)
         add_test(
@@ -206,6 +244,10 @@ function(add_warpx_test
             # test cleanup depends on test analysis
             set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.analysis")
         endif()
+        if(checksum)
+            # test cleanup depends on test analysis
+            set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.checksum")
+        endif()
     endif()
 
     # Do we depend on another test?
@@ -215,6 +257,9 @@ function(add_warpx_test
         if(analysis)
             set_property(TEST ${name}.run APPEND PROPERTY DEPENDS "${dependency}.analysis")
         endif()
+        if(checksum)
+            set_property(TEST ${name}.run APPEND PROPERTY DEPENDS "${dependency}.checksum")
+        endif()
         if(WarpX_TEST_CLEANUP)
             # do not clean up dependency test before current test is completed
             set_property(TEST ${dependency}.cleanup APPEND PROPERTY DEPENDS "${name}.cleanup")

Examples/Physics_applications/beam_beam_collision/CMakeLists.txt

@@ -6,8 +6,8 @@ add_warpx_test(
     3  # dims
     2  # nprocs
     inputs_test_3d_beam_beam_collision  # inputs
-    analysis_default_openpmd_regression.py  # analysis
-    diags/diag1/  # output
+    OFF  # analysis
+    "analysis_default_regression.py --path diags/diag1/"  # checksum
     OFF  # dependency
 )
 label_warpx_test(test_3d_beam_beam_collision slow)

Examples/Physics_applications/beam_beam_collision/analysis_default_regression.py

@@ -0,0 +1 @@
+../../analysis_default_regression.py