RevathiJambunathan · hklion · Sep 8, 2023 · Sep 8, 2023 · Sep 8, 2023 · Sep 8, 2023
diff --git a/.clang-tidy b/.clang-tidy
@@ -17,15 +17,9 @@ Checks: '-*,
     google-build-explicit-make-pair,
     google-build-namespaces,
     google-global-names-in-headers,
-    misc-const-correctness,
-    misc-misleading-bidirectional,
-    misc-misleading-identifier,
-    misc-misplaced-const,
-    misc-uniqueptr-reset-release,
-    misc-unused-alias-decls,
-    misc-unused-parameters,
-    misc-unused-using-decls,
-        -misc-definitions-in-headers,
+    misc-*,
+        -misc-no-recursion,
+        -misc-non-private-member-variables-in-classes,
     modernize-avoid-bind,
     modernize-concat-nested-namespaces,
     modernize-deprecated-headers,
@@ -82,5 +76,7 @@ Checks: '-*,
 CheckOptions:
 - key:          modernize-pass-by-value.ValuesOnly
   value:        'true'
+- key:          misc-definitions-in-headers.HeaderFileExtensions
+  value:        "H,"
 
 HeaderFilterRegex: 'Source[a-z_A-Z0-9\/]+\.H$'
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -111,7 +111,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 23.09 && cd -
+        cd ../amrex && git checkout --detach 2e99628138df3b5b0ecf50b0c1201d5547f821a0 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
   build_nvhpc21-11-nvcc:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -372,6 +372,7 @@ if(WarpX_PYTHON)
 endif()
 
 add_subdirectory(Source/ablastr)
+
 if(WarpX_LIB)
     add_subdirectory(Source/AcceleratorLattice)
     add_subdirectory(Source/BoundaryConditions)
@@ -380,6 +381,7 @@ if(WarpX_LIB)
     add_subdirectory(Source/Evolve)
     add_subdirectory(Source/FieldSolver)
     add_subdirectory(Source/Filter)
+    add_subdirectory(Source/Fluids)
     add_subdirectory(Source/Initialization)
     add_subdirectory(Source/Laser)
     add_subdirectory(Source/Parallelization)

diff --git a/Docs/source/glossary.rst b/Docs/source/glossary.rst
@@ -18,6 +18,7 @@ Abbreviations
 * **BC:** boundary condition (of a simulation)
 * **BCK:** `Benkler-Chavannes-Kuster <https://ieeexplore.ieee.org/document/1638381>`__ method, a stabilization technique for small cells in the electromagnetic solver
 * **BTD:** backtransformed diagnosics, a method to collect data for analysis from a *boosted frame* simulation
+* **CEX:** charge-exchange collisions
 * **CFL:** the Courant-Friedrichs-Lewy condition, a numerical parameter for the numerical convergence of PDE solvers
 * **CI:** continuous integration, automated tests that we perform before a proposed code-change is accepted; see PR
 * **CPU:** `central processing unit <https://en.wikipedia.org/wiki/Central_processing_unit>`__, we usual mean a socket or generally the host-side of a computer (compared to the accelerator, e.g. GPU)
@@ -57,6 +58,7 @@ Abbreviations
 * **RPA:** radiation-pressure acceleration (of protons/ions), e.g. hole-boring (HB) or light-sail (LS) acceleration
 * **RZ:** for the coordinate system ``r-z`` in cylindrical geometry; we use "RZ" when we refer to quasi-cylindrical geometry, decomposed in azimuthal modes (see details `here <https://fbpic.github.io/overview/pic_algorithm.html#cylindrical-grid-with-azimuthal-decomposition>`__)
 * **SENSEI:** `Scalable in situ analysis and visualization <https://sensei-insitu.org/>`__, light weight framework for in situ data analysis offering access to multiple visualization and analysis backends
+* **SEE:** secondary electron emission
 * **TNSA:** target-normal sheet acceleration (of protons/ions)
 
 Terms

diff --git a/Docs/source/index.rst b/Docs/source/index.rst
@@ -116,6 +116,7 @@ Theory
    theory/input_output
    theory/collisions
    theory/kinetic_fluid_hybrid_model
+   theory/cold_fluid_model
 
 Development
 -----------

diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst
@@ -75,7 +75,7 @@ Conda (Linux/macOS/Windows)
 
       .. code-block:: bash
 
-         conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv
+         conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv
          conda activate warpx-cpu-mpich-dev
 
          # compile WarpX with -DWarpX_MPI=ON
@@ -85,7 +85,7 @@ Conda (Linux/macOS/Windows)
 
       .. code-block:: bash
 
-         conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv
+         conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv
          conda activate warpx-cpu-dev
 
          # compile WarpX with -DWarpX_MPI=OFF
@@ -107,6 +107,14 @@ For OpenMP support, you will further need:
 
          conda install -c conda-forge llvm-openmp
 
+For Nvidia CUDA GPU support, you will need to have `a recent CUDA driver installed <https://developer.nvidia.com/cuda-downloads>`__ or you can lower the CUDA version of `the Nvidia cuda package <https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#conda-installation>`__ and `conda-forge to match your drivers <https://docs.cupy.dev/en/stable/install.html#install-cupy-from-conda-forge>`__ and then add these packages:
+
+.. code-block:: bash
+
+   conda install -c nvidia -c conda-forge cuda cupy
+
+More info for `CUDA-enabled ML packages <https://twitter.com/jeremyphoward/status/1697435241152127369>`__.
+
 
 Spack (Linux/macOS)
 -------------------

diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst
@@ -3,93 +3,178 @@
 Lassen (LLNL)
 =============
 
-The `Lassen V100 GPU cluster <https://hpc.llnl.gov/hardware/platforms/lassen>`_ is located at LLNL.
+The `Lassen V100 GPU cluster <https://hpc.llnl.gov/hardware/platforms/lassen>`__ is located at LLNL.
 
 
 Introduction
 ------------
 
 If you are new to this system, **please see the following resources**:
 
-* `LLNL user account <https://lc.llnl.gov/lorenz/mylc/mylc.cgi>`_
-* `Lassen user guide <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system>`_
-* Batch system: `LSF <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system#batch-system>`_
-* `Production directories <https://hpc.llnl.gov/hardware/file-systems>`_:
+* `LLNL user account <https://lc.llnl.gov/lorenz/mylc/mylc.cgi>`__ (login required)
+* `Lassen user guide <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system>`__
+* Batch system: `LSF <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system#batch-system>`__
+* `Jupyter service <https://lc.llnl.gov/jupyter>`__ (`documentation <https://lc.llnl.gov/confluence/display/LC/JupyterHub+and+Jupyter+Notebook>`__, login required)
+* `Production directories <https://hpc.llnl.gov/hardware/file-systems>`__:
 
   * ``/p/gpfs1/$(whoami)``: personal directory on the parallel filesystem
   * Note that the ``$HOME`` directory and the ``/usr/workspace/$(whoami)`` space are NFS mounted and *not* suitable for production quality data generation.
 
 
-Installation
-------------
+Login
+-----
 
-Use the following commands to download the WarpX source code and switch to the correct branch:
+.. note::
 
-.. code-block:: bash
+   Lassen is currently transitioning to RHEL8.
+   During this transition, first SSH into lassen and then ``ssh eatoss4`` next to work with the updated RHEL8/TOSS4 nodes.
 
-   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+   Approximately October 2023, the new software environment on these nodes will be the new default.
 
-We use the following modules and environments on the system (``$HOME/lassen_warpx.profile``).
 
-.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_warpx.profile.example
-   :language: bash
-   :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen_warpx.profile.example``.
+.. _building-lassen-preparation:
+
+Preparation
+-----------
+
+Use the following commands to download the WarpX source code:
+
+.. code-block:: bash
 
-We recommend to store the above lines in a file, such as ``$HOME/lassen_warpx.profile``, and load it into your shell after a login:
+   git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx
+
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
+Create it now:
 
 .. code-block:: bash
 
-   source $HOME/lassen_warpx.profile
+   cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
 
-And since Lassen does not yet provide a module for them, install ADIOS2, BLAS++ and LAPACK++:
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
+      :language: bash
+
+Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``.
+Enter the edit mode by typing ``i`` and edit line 2 to read:
 
 .. code-block:: bash
 
-   # c-blosc (I/O compression)
-   git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
-   rm -rf src/c-blosc-lassen-build
-   cmake -S src/c-blosc -B src/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/c-blosc-1.21.1
-   cmake --build src/c-blosc-lassen-build --target install --parallel 16
-
-   # HDF5
-   git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5
-   rm -rf src/hdf5-lassen-build
-   cmake -S src/hdf5 -B src/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/hdf5-1.14.1.2
-   cmake --build src/hdf5-lassen-build --target install --parallel 16
-
-   # ADIOS2
-   git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2
-   rm -rf src/adios2-lassen-build
-   cmake -S src/adios2 -B src/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.8.3
-   cmake --build src/adios2-lassen-build --target install -j 16
-
-   # BLAS++ (for PSATD+RZ)
-   git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp
-   rm -rf src/blaspp-lassen-build
-   cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master
-   cmake --build src/blaspp-lassen-build --target install --parallel 16
-
-   # LAPACK++ (for PSATD+RZ)
-   git clone https://github.com/icl-utk-edu/lapackpp.git src/lapackpp
-   rm -rf src/lapackpp-lassen-build
-   CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so
-   cmake --build src/lapackpp-lassen-build --target install --parallel 16
-
-Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile:
+   export proj="nsldt"
+
+Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+
+.. important::
+
+   Now, and as the first step on future logins to lassen, activate these environment settings:
+
+   .. code-block:: bash
+
+      source $HOME/lassen_v100_warpx.profile
+
+Finally, since lassen does not yet provide software modules for some of our dependencies, install them once:
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
+   source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh
+      :language: bash
+
+.. dropdown:: AI/ML Dependencies (Optional)
+   :animate: fade-in-slide-down
+
+   If you plan to run AI/ML workflows depending on pyTorch, run the next step as well.
+   This will take a while and should be skipped if not needed.
 
+   .. code-block:: bash
+
+      runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
+
+   .. dropdown:: Script Details
+      :color: light
+      :icon: info
+      :animate: fade-in-slide-down
+
+      .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh
+         :language: bash
+
+   For `optimas dependencies <https://github.com/optimas-org/optimas>`__ (incl. scikit-learn), plan another hour of build time:
+
+   .. code-block:: bash
+
+      python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt
+
+
+.. _building-lassen-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands <building-cmake>` to compile the application executable:
+
+.. code-block:: bash
+
+   cd /usr/workspace/${USER}/lassen/src/warpx
    rm -rf build_lassen
-   cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_DIMS="1;2;RZ;3" -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON
-   cmake --build build_lassen -j 10
 
-The other :ref:`general compile-time options <building-cmake>` apply as usual.
+   cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_lassen -j 8
+
+The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``.
+Additionally, the following commands will install WarpX as a Python module:
+
+.. code-block:: bash
+
+   rm -rf build_lassen_py
+
+   cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_lassen_py -j 8 --target pip_install
+
+Now, you can :ref:`submit lassen compute jobs <running-cpp-lassen>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
+Or, you can use the WarpX executables to submit lassen jobs (:ref:`example inputs <usage-examples>`).
+For executables, you can reference their location in your :ref:`job script <running-cpp-lassen>` or copy them to a location in ``$PROJWORK/$proj/``.
 
-**That's it!**
-WarpX executables for 1D, 2D, RZ and 3D are now in ``build_lassen/bin/`` and :ref:`can be run <running-cpp-lassen>` with the respective :ref:`example inputs files <usage-examples>`.
-Most people execute the binary directly or copy it out to a location in ``/p/gpfs1/$(whoami)``.
+
+.. _building-lassen-update:
+
+Update WarpX & Dependencies
+---------------------------
+
+If you already installed WarpX in the past and want to update it, start by getting the latest source code:
+
+.. code-block:: bash
+
+   cd /usr/workspace/${USER}/lassen/src/warpx
+
+   # read the output of this command - does it look ok?
+   git status
+
+   # get the latest WarpX source code
+   git fetch
+   git pull
+
+   # read the output of these commands - do they look ok?
+   git status
+   git log     # press q to exit
+
+And, if needed,
+
+- :ref:`update the lassen_v100_warpx.profile file <building-lassen-preparation>`,
+- log out and into the system, activate the now updated environment profile as usual,
+- :ref:`execute the dependency install scripts <building-lassen-preparation>`.
+
+As a last step, clean the build directory ``rm -rf /usr/workspace/${USER}/lassen/src/warpx/build_lassen`` and rebuild WarpX.
 
 
 .. _running-cpp-lassen:
@@ -146,3 +231,6 @@ Known System Issues
    .. code-block:: bash
 
       export OMPI_MCA_coll_ibm_skip_allgatherv=true
+
+As part of the same `CORAL acquisition program <https://doi.org/10.1109/SC.2018.00055>`__, Lassen is very similar to the design of Summit (OLCF).
+Thus, when encountering new issues it is worth checking also the :ref:`known Summit issues and work-arounds <building-summit-issues>`.