Merge branch 'ECP-WarpX:development' into add_feature_projection_div_…

…cleaner
clarkse · Jul 22, 2024 · d9de8b5 · d9de8b5
2 parents 7baa18c + 7dad8f3
commit d9de8b5
Show file tree

Hide file tree

Showing 40 changed files with 391 additions and 1,375 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -115,7 +115,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 24.07 && cd -
+        cd ../amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -146,10 +146,19 @@ set_default_build_type("Release")
 
 # Option to enable interprocedural optimization
 # (also know as "link-time optimization" or "whole program optimization")
-option(WarpX_IPO                                "Compile WarpX with interprocedural optimization (will take more time)" OFF)
+set(_WarpX_IPO_DEFAULT OFF)
+set(_WarpX_PYTHON_IPO_DEFAULT ON)
+if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
+    set(_WarpX_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION})
+    set(_WarpX_PYTHON_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION})
+endif()
+option(WarpX_IPO
+    "Compile WarpX with interprocedural optimization (will take more time)"
+    ${_WarpX_IPO_DEFAULT}
+)
 option(WarpX_PYTHON_IPO
     "Compile Python bindings with interprocedural optimization (IPO) / link-time optimization (LTO)"
-    ON
+    ${_WarpX_PYTHON_IPO_DEFAULT}
 )
 
 set(pyWarpX_VERSION_INFO "" CACHE STRING
@@ -455,7 +464,7 @@ endif()
 
 # Interprocedural optimization (IPO) / Link-Time Optimization (LTO)
 if(WarpX_IPO)
-    enable_IPO("${_ALL_TARGETS}")
+    warpx_enable_IPO("${_ALL_TARGETS}")
 endif()
 
 # link dependencies
@@ -488,7 +497,13 @@ foreach(D IN LISTS WarpX_DIMS)
     if(WarpX_PYTHON)
         target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras)
         if(WarpX_PYTHON_IPO)
-            target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto)
+            if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
+                warpx_enable_IPO(pyWarpX_${SD})
+            else()
+                # conditionally defined target in pybind11
+                # https://github.com/pybind/pybind11/blob/v2.12.0/tools/pybind11Common.cmake#L397-L403
+                target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto)
+            endif()
         endif()
     endif()
 

diff --git a/Docs/requirements.txt b/Docs/requirements.txt
@@ -13,7 +13,7 @@ openpmd-viewer  # for checksumAPI
 
 # PICMI API docs
 # note: keep in sync with version in ../requirements.txt
-picmistandard==0.28.0
+picmistandard==0.29.0
 # for development against an unreleased PICMI version, use:
 # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python
 

diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst
@@ -24,27 +24,9 @@ If you are new to this system, **please see the following resources**:
 Login
 -----
 
-.. tab-set::
-
-   .. tab-item:: TOSS4 (RHEL8)
-
-      Lassen is currently transitioning to RHEL8.
-      During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes.
-
-      .. code-block:: bash
-
-         ssh lassen.llnl.gov
-         ssh eatoss4
-
-      Approximately October/November 2023, the new software environment on these nodes will be the new default.
-
-   .. tab-item:: TOSS3 (RHEL7)
-
-      .. code-block:: bash
-
-         ssh lassen.llnl.gov
+.. code-block:: bash
 
-      Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well.
+   ssh lassen.llnl.gov
 
 
 .. _building-lassen-preparation:
@@ -58,135 +40,54 @@ Use the following commands to download the WarpX source code:
 
    git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx
 
-.. tab-set::
-
-   .. tab-item:: TOSS4 (RHEL8)
-
-      We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
-      Create it now:
-
-      .. code-block:: bash
-
-         cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
-
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
-
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
-            :language: bash
-
-      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
-      For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``.
-      Enter the edit mode by typing ``i`` and edit line 2 to read:
-
-      .. code-block:: bash
-
-         export proj="nsldt"
-
-      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
-
-      .. important::
-
-         Now, and as the first step on future logins to lassen, activate these environment settings:
-
-         .. code-block:: bash
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``.
+Create it now:
 
-            source $HOME/lassen_v100_warpx.profile
-
-   .. tab-item:: TOSS3 (RHEL7)
-
-      We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``.
-      Create it now:
-
-      .. code-block:: bash
+.. code-block:: bash
 
-         cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile
+   cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile
 
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
 
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example
-            :language: bash
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example
+      :language: bash
 
-      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
-      For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``.
-      Enter the edit mode by typing ``i`` and edit line 2 to read:
+Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``.
+Enter the edit mode by typing ``i`` and edit line 2 to read:
 
-      .. code-block:: bash
+.. code-block:: bash
 
-         export proj="nsldt"
+   export proj="nsldt"
 
-      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
 
-      .. important::
+.. important::
 
-         Now, and as the first step on future logins to lassen, activate these environment settings:
+   Now, and as the first step on future logins to lassen, activate these environment settings:
 
-         .. code-block:: bash
+   .. code-block:: bash
 
-            source $HOME/lassen_v100_warpx_toss3.profile
+      source $HOME/lassen_v100_warpx_toss3.profile
 
 Finally, since lassen does not yet provide software modules for some of our dependencies, install them once:
 
-.. tab-set::
-
-   .. tab-item:: TOSS4 (RHEL8)
-
-      .. code-block:: bash
-
-         bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
-         source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
-
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
-
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh
-            :language: bash
-
-      .. dropdown:: AI/ML Dependencies (Optional)
-         :animate: fade-in-slide-down
-
-         If you plan to run AI/ML workflows depending on pyTorch, run the next step as well.
-         This will take a while and should be skipped if not needed.
-
-         .. code-block:: bash
-
-            runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
-
-         .. dropdown:: Script Details
-            :color: light
-            :icon: info
-            :animate: fade-in-slide-down
-
-            .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh
-               :language: bash
-
-         For `optimas dependencies <https://github.com/optimas-org/optimas>`__ (incl. scikit-learn), plan another hour of build time:
-
-         .. code-block:: bash
-
-            python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt
-
-   .. tab-item:: TOSS3 (RHEL7)
+.. code-block:: bash
 
-      .. code-block:: bash
+   bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
+   source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate
 
-         bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
-         source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
 
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
+      :language: bash
 
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
-            :language: bash
 
 .. _building-lassen-compilation:
 

diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst
@@ -500,7 +500,7 @@ Domain Boundary Conditions
     * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method.
 
 * ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`)
-    Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers
+    Gives the value of the electric potential, in Volts, at the boundaries, for ``pec`` boundaries. With electrostatic solvers
     (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential
     in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver),
     setting these variables will trigger an electrostatic solve at ``t=0``, to compute the initial
@@ -603,7 +603,7 @@ Whether the embedded boundary is defined with an analytical function or an STL f
 additionally define the electric potential at the embedded boundary with an analytical function:
 
 * ``warpx.eb_potential(x,y,z,t)`` (`string`)
-    Gives the value of the electric potential at the surface of the embedded boundary,
+    Gives the value of the electric potential, in Volts, at the surface of the embedded boundary,
     as a function of  `x`, `y`, `z` and `t`. With electrostatic solvers (i.e., with
     ``warpx.do_electrostatic = ...``), this is used in order to compute the potential
     in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver),
@@ -1156,8 +1156,8 @@ Particle initialization
     * ``gaussian_parse_momentum_function``: Gaussian momentum distribution where the mean and the standard deviation are given by functions of position in the input file.
       Both are assumed to be non-relativistic.
       The mean is the normalized momentum, :math:`u_m = \gamma v_m/c`.
-      The standard deviation is normalized, :math:`u_th = v_th/c`.
-      For example, this might be `u_th = sqrt(T*q_e/mass)/clight` given the temperature (in eV) and mass.
+      The standard deviation is normalized, :math:`u_{th} = v_{th}/c`.
+      For example, this might be ``u_th = sqrt(T*q_e/mass)/clight`` given the temperature (in eV) and mass.
       It requires the following arguments:
 
       * ``<species_name>.momentum_function_ux_m(x,y,z)``: mean :math:`u_{x}`

diff --git a/Examples/Tests/collision/analysis_collision_3d.py b/Examples/Tests/collision/analysis_collision_3d.py
@@ -111,4 +111,4 @@
                                           dim, species_name)
 
 test_name = os.path.split(os.getcwd())[1]
-checksumAPI.evaluate_checksum(test_name, fn, do_particles=False)
+checksumAPI.evaluate_checksum(test_name, fn)
diff --git a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py
@@ -47,7 +47,9 @@
 
 sigma_th = 2.603e-18
 error = np.abs(sigma-sigma_th)/sigma_th
-tolerance = 1e-2
+tolerance = 2e-2
+print('error = ', error)
+print('tolerance = ', tolerance)
 assert error < tolerance
 
 # Compare checksums with benchmark

diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies
@@ -8,6 +8,8 @@
 #     plasma simulations. Journal of Computational Physics, 388, pp.439-453.
 #     DOI: https://doi.org/10.1016/j.jcp.2019.03.020
 
+warpx.random_seed = 1034958209
+
 # algo
 algo.particle_shape = 3
 

diff --git a/Python/setup.py b/Python/setup.py
@@ -59,7 +59,7 @@
       package_dir = {'pywarpx': 'pywarpx'},
       description = """Wrapper of WarpX""",
       package_data = package_data,
-      install_requires = ['numpy', 'picmistandard==0.28.0', 'periodictable'],
+      install_requires = ['numpy', 'picmistandard==0.29.0', 'periodictable'],
       python_requires = '>=3.8',
       zip_safe=False
 )
diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json
@@ -1,34 +1,34 @@
 {
   "lev=0": {
     "rho": 0.0,
-    "rho_deuterium": 8203144355.71195,
-    "rho_helium3": 10.368009592276463
+    "rho_deuterium": 8203144355.767546,
+    "rho_helium3": 10.312416024595505
   },
   "neutron": {
-    "particle_momentum_x": 2.2543499835759282e-15,
-    "particle_momentum_y": 2.2526527390783875e-15,
-    "particle_momentum_z": 2.2619641737859965e-15,
-    "particle_position_x": 61.961041864660686,
-    "particle_position_y": 61.78141653674165,
-    "particle_position_z": 61.741022731492514,
-    "particle_weight": 505562702.7678892
+    "particle_momentum_x": 2.260586487910896e-15,
+    "particle_momentum_y": 2.2564723094408887e-15,
+    "particle_momentum_z": 2.263164930227645e-15,
+    "particle_position_x": 62.03084683414219,
+    "particle_position_y": 61.95011296352698,
+    "particle_position_z": 62.02404756675538,
+    "particle_weight": 502851860.91505826
   },
   "deuterium": {
-    "particle_momentum_x": 1.3370046499332103e-14,
-    "particle_momentum_y": 1.3364310231320824e-14,
-    "particle_momentum_z": 1.3372728873714894e-14,
-    "particle_position_x": 2560.1613417364665,
-    "particle_position_y": 2560.082464065988,
-    "particle_position_z": 2560.0018477161034,
-    "particle_weight": 7.999999989888742e+17
+    "particle_momentum_x": 1.3380470537895298e-14,
+    "particle_momentum_y": 1.3367685546044215e-14,
+    "particle_momentum_z": 1.3372922616599391e-14,
+    "particle_position_x": 2559.782513518522,
+    "particle_position_y": 2559.8747993471684,
+    "particle_position_z": 2560.2864832238383,
+    "particle_weight": 7.999999989942968e+17
   },
   "helium3": {
-    "particle_momentum_x": 2.2749239620327265e-15,
-    "particle_momentum_y": 2.268697031603961e-15,
-    "particle_momentum_z": 2.278045756364995e-15,
-    "particle_position_x": 61.961041864660686,
-    "particle_position_y": 61.78141653674165,
-    "particle_position_z": 61.741022731492514,
-    "particle_weight": 505562702.7678892
+    "particle_momentum_x": 2.278275200450756e-15,
+    "particle_momentum_y": 2.2738610200497133e-15,
+    "particle_momentum_z": 2.2792408973056887e-15,
+    "particle_position_x": 62.03084683414219,
+    "particle_position_y": 61.95011296352698,
+    "particle_position_z": 62.02404756675538,
+    "particle_weight": 502851860.91505826
   }
-}
+}