diff --git a/.clang-tidy b/.clang-tidy index 2d495081c8d..42f4fc36065 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -17,15 +17,9 @@ Checks: '-*, google-build-explicit-make-pair, google-build-namespaces, google-global-names-in-headers, - misc-const-correctness, - misc-misleading-bidirectional, - misc-misleading-identifier, - misc-misplaced-const, - misc-uniqueptr-reset-release, - misc-unused-alias-decls, - misc-unused-parameters, - misc-unused-using-decls, - -misc-definitions-in-headers, + misc-*, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, modernize-avoid-bind, modernize-concat-nested-namespaces, modernize-deprecated-headers, @@ -82,5 +76,7 @@ Checks: '-*, CheckOptions: - key: modernize-pass-by-value.ValuesOnly value: 'true' +- key: misc-definitions-in-headers.HeaderFileExtensions + value: "H," HeaderFilterRegex: 'Source[a-z_A-Z0-9\/]+\.H$' diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 17b8c259f0a..03a3044848f 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -111,7 +111,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 23.09 && cd - + cd ../amrex && git checkout --detach 2e99628138df3b5b0ecf50b0c1201d5547f821a0 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 build_nvhpc21-11-nvcc: diff --git a/CMakeLists.txt b/CMakeLists.txt index ca98c214831..3d37c463cd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -372,6 +372,7 @@ if(WarpX_PYTHON) endif() add_subdirectory(Source/ablastr) + if(WarpX_LIB) add_subdirectory(Source/AcceleratorLattice) add_subdirectory(Source/BoundaryConditions) @@ -380,6 +381,7 @@ if(WarpX_LIB) add_subdirectory(Source/Evolve) add_subdirectory(Source/FieldSolver) add_subdirectory(Source/Filter) + add_subdirectory(Source/Fluids) add_subdirectory(Source/Initialization) add_subdirectory(Source/Laser) add_subdirectory(Source/Parallelization) diff --git a/Docs/source/glossary.rst b/Docs/source/glossary.rst index cb376af481f..131b3a63ae7 100644 --- a/Docs/source/glossary.rst +++ b/Docs/source/glossary.rst @@ -18,6 +18,7 @@ Abbreviations * **BC:** boundary condition (of a simulation) * **BCK:** `Benkler-Chavannes-Kuster `__ method, a stabilization technique for small cells in the electromagnetic solver * **BTD:** backtransformed diagnosics, a method to collect data for analysis from a *boosted frame* simulation +* **CEX:** charge-exchange collisions * **CFL:** the Courant-Friedrichs-Lewy condition, a numerical parameter for the numerical convergence of PDE solvers * **CI:** continuous integration, automated tests that we perform before a proposed code-change is accepted; see PR * **CPU:** `central processing unit `__, we usual mean a socket or generally the host-side of a computer (compared to the accelerator, e.g. GPU) @@ -57,6 +58,7 @@ Abbreviations * **RPA:** radiation-pressure acceleration (of protons/ions), e.g. hole-boring (HB) or light-sail (LS) acceleration * **RZ:** for the coordinate system ``r-z`` in cylindrical geometry; we use "RZ" when we refer to quasi-cylindrical geometry, decomposed in azimuthal modes (see details `here `__) * **SENSEI:** `Scalable in situ analysis and visualization `__, light weight framework for in situ data analysis offering access to multiple visualization and analysis backends +* **SEE:** secondary electron emission * **TNSA:** target-normal sheet acceleration (of protons/ions) Terms diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 2209b360e26..7fc2a1c5833 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -116,6 +116,7 @@ Theory theory/input_output theory/collisions theory/kinetic_fluid_hybrid_model + theory/cold_fluid_model Development ----------- diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index a198712cc01..7bd6c7dc8d5 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -75,7 +75,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON @@ -85,7 +85,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv + conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv conda activate warpx-cpu-dev # compile WarpX with -DWarpX_MPI=OFF @@ -107,6 +107,14 @@ For OpenMP support, you will further need: conda install -c conda-forge llvm-openmp +For Nvidia CUDA GPU support, you will need to have `a recent CUDA driver installed `__ or you can lower the CUDA version of `the Nvidia cuda package `__ and `conda-forge to match your drivers `__ and then add these packages: + +.. code-block:: bash + + conda install -c nvidia -c conda-forge cuda cupy + +More info for `CUDA-enabled ML packages `__. + Spack (Linux/macOS) ------------------- diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 1f03a3c117f..5726254652a 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -3,7 +3,7 @@ Lassen (LLNL) ============= -The `Lassen V100 GPU cluster `_ is located at LLNL. +The `Lassen V100 GPU cluster `__ is located at LLNL. Introduction @@ -11,85 +11,170 @@ Introduction If you are new to this system, **please see the following resources**: -* `LLNL user account `_ -* `Lassen user guide `_ -* Batch system: `LSF `_ -* `Production directories `_: +* `LLNL user account `__ (login required) +* `Lassen user guide `__ +* Batch system: `LSF `__ +* `Jupyter service `__ (`documentation `__, login required) +* `Production directories `__: * ``/p/gpfs1/$(whoami)``: personal directory on the parallel filesystem * Note that the ``$HOME`` directory and the ``/usr/workspace/$(whoami)`` space are NFS mounted and *not* suitable for production quality data generation. -Installation ------------- +Login +----- -Use the following commands to download the WarpX source code and switch to the correct branch: +.. note:: -.. code-block:: bash + Lassen is currently transitioning to RHEL8. + During this transition, first SSH into lassen and then ``ssh eatoss4`` next to work with the updated RHEL8/TOSS4 nodes. - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + Approximately October 2023, the new software environment on these nodes will be the new default. -We use the following modules and environments on the system (``$HOME/lassen_warpx.profile``). -.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_warpx.profile.example - :language: bash - :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen_warpx.profile.example``. +.. _building-lassen-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash -We recommend to store the above lines in a file, such as ``$HOME/lassen_warpx.profile``, and load it into your shell after a login: + git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx + +We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. +Create it now: .. code-block:: bash - source $HOME/lassen_warpx.profile + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down -And since Lassen does not yet provide a module for them, install ADIOS2, BLAS++ and LAPACK++: + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example + :language: bash + +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: .. code-block:: bash - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-lassen-build - cmake -S src/c-blosc -B src/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/c-blosc-1.21.1 - cmake --build src/c-blosc-lassen-build --target install --parallel 16 - - # HDF5 - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 - rm -rf src/hdf5-lassen-build - cmake -S src/hdf5 -B src/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/hdf5-1.14.1.2 - cmake --build src/hdf5-lassen-build --target install --parallel 16 - - # ADIOS2 - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-lassen-build - cmake -S src/adios2 -B src/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.8.3 - cmake --build src/adios2-lassen-build --target install -j 16 - - # BLAS++ (for PSATD+RZ) - git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp - rm -rf src/blaspp-lassen-build - cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master - cmake --build src/blaspp-lassen-build --target install --parallel 16 - - # LAPACK++ (for PSATD+RZ) - git clone https://github.com/icl-utk-edu/lapackpp.git src/lapackpp - rm -rf src/lapackpp-lassen-build - CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so - cmake --build src/lapackpp-lassen-build --target install --parallel 16 - -Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: + export proj="nsldt" + +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). + +.. important:: + + Now, and as the first step on future logins to lassen, activate these environment settings: + + .. code-block:: bash + + source $HOME/lassen_v100_warpx.profile + +Finally, since lassen does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - cd $HOME/src/warpx + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh + source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh + :language: bash + +.. dropdown:: AI/ML Dependencies (Optional) + :animate: fade-in-slide-down + + If you plan to run AI/ML workflows depending on pyTorch, run the next step as well. + This will take a while and should be skipped if not needed. + .. code-block:: bash + + runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh + :language: bash + + For `optimas dependencies `__ (incl. scikit-learn), plan another hour of build time: + + .. code-block:: bash + + python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt + + +.. _building-lassen-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. code-block:: bash + + cd /usr/workspace/${USER}/lassen/src/warpx rm -rf build_lassen - cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_DIMS="1;2;RZ;3" -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON - cmake --build build_lassen -j 10 -The other :ref:`general compile-time options ` apply as usual. + cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_lassen -j 8 + +The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``. +Additionally, the following commands will install WarpX as a Python module: + +.. code-block:: bash + + rm -rf build_lassen_py + + cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_lassen_py -j 8 --target pip_install + +Now, you can :ref:`submit lassen compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit lassen jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PROJWORK/$proj/``. -**That's it!** -WarpX executables for 1D, 2D, RZ and 3D are now in ``build_lassen/bin/`` and :ref:`can be run ` with the respective :ref:`example inputs files `. -Most people execute the binary directly or copy it out to a location in ``/p/gpfs1/$(whoami)``. + +.. _building-lassen-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd /usr/workspace/${USER}/lassen/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the lassen_v100_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf /usr/workspace/${USER}/lassen/src/warpx/build_lassen`` and rebuild WarpX. .. _running-cpp-lassen: @@ -146,3 +231,6 @@ Known System Issues .. code-block:: bash export OMPI_MCA_coll_ibm_skip_allgatherv=true + +As part of the same `CORAL acquisition program `__, Lassen is very similar to the design of Summit (OLCF). +Thus, when encountering new issues it is worth checking also the :ref:`known Summit issues and work-arounds `. diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index 8902ec2cf45..de7c5ace848 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -11,52 +11,138 @@ Introduction If you are new to this system, **please see the following resources**: -* `LLNL user account `_ +* `LLNL user account `__ (login required) * `Quartz user guide `_ * Batch system: `Slurm `_ +* `Jupyter service `__ (`documentation `__, login required) * `Production directories `_: * ``/p/lustre1/$(whoami)`` and ``/p/lustre2/$(whoami)``: personal directory on the parallel filesystem * Note that the ``$HOME`` directory and the ``/usr/workspace/$(whoami)`` space are NFS mounted and *not* suitable for production quality data generation. -Installation ------------- +.. _building-quartz-preparation: + +Preparation +----------- -Use the following commands to download the WarpX source code and switch to the correct branch: +Use the following commands to download the WarpX source code: .. code-block:: bash git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx -We use the following modules and environments on the system (``$HOME/quartz_warpx.profile``). +We use system software modules, add environment hints and further dependencies via the file ``$HOME/quartz_warpx.profile``. +Create it now: -.. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz_warpx.profile.example - :language: bash - :caption: You can copy this file from ``Tools/machines/quartz-llnl/quartz_warpx.profile.example``. +.. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example $HOME/quartz_warpx.profile + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example + :language: bash + +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``tps``, then run ``vi $HOME/quartz_warpx.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: + +.. code-block:: bash + + export proj="tps" + +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). + +.. important:: + + Now, and as the first step on future logins to Quartz, activate these environment settings: + + .. code-block:: bash + + source $HOME/quartz_warpx.profile + +Finally, since Quartz does not yet provide software modules for some of our dependencies, install them once: + +.. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/quartz-llnl/install_dependencies.sh + source /usr/workspace/${USER}/quartz/venvs/warpx-quartz/bin/activate -We recommend to store the above lines in a file, such as ``$HOME/quartz_warpx.profile``, and load it into your shell after a login: +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/quartz-llnl/install_dependencies.sh + :language: bash + + +.. _building-quartz-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_quartz + + cmake -S . -B build_quartz -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_quartz -j 6 + +The WarpX application executables are now in ``$HOME/src/warpx/build_quartz/bin/``. +Additionally, the following commands will install WarpX as a Python module: .. code-block:: bash - source $HOME/quartz_warpx.profile + rm -rf build_quartz_py + + cmake -S . -B build_quartz_py -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_quartz_py -j 6 --target pip_install + +Now, you can :ref:`submit Quartz compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit Quartz jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PROJWORK/$proj/``. + + +.. _building-quartz-update: -Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: .. code-block:: bash cd $HOME/src/warpx - rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 6 + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the quartz_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. -The other :ref:`general compile-time options ` apply as usual. +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_quartz`` and rebuild WarpX. -**That's it!** -A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. -Most people execute the binary directly or copy it out to a location in ``/p/lustre1/$(whoami)``. +.. _running-cpp-quartz: Running ------- diff --git a/Docs/source/theory/cold_fluid_model.rst b/Docs/source/theory/cold_fluid_model.rst new file mode 100644 index 00000000000..0a98ede772f --- /dev/null +++ b/Docs/source/theory/cold_fluid_model.rst @@ -0,0 +1,135 @@ +.. _theory-cold-fluid-model: + +Cold Relativistic Fluid Model +============================= + +An alternate to the representation of the plasma as macroparticles, is the cold relativistic fluid model. +The cold relativistic fluid model is typically faster to compute than +particles and useful to replace particles when kinetic effects are negligible. This +can be done for certain parts of the plasma, such as the background plasma, while still +representing particle beams as a group of macroparticles. The two models then couple through +Maxwell's equations. + +In the cold limit (zero internal temperature and pressure) of a relativistic plasma, the Maxwell-Fluid +equations govern the plasma evolution. The fluid equations per species, ``s``, are given by, + +.. math:: + + \frac{\partial N_s}{\partial t} + \nabla \cdot (N_s\mathbf{V}_s) &= 0 \\ + \frac{\partial (N\mathbf{U})_s}{\partial t} + \nabla \cdot ((N\mathbf{U})_s\mathbf{V}_s) &= \frac{q_sN_s}{m_s}(\mathbf{E}_s + \mathbf{V}_s \times \mathbf{B}_s). + +Where the fields are updated via Maxwell's equations, + +.. math:: + + \nabla \cdot \mathbf{E} &= \frac{\rho}{\varepsilon_0} \\ + \nabla \cdot \mathbf{B} &= 0 \\ + \nabla \times \mathbf{E} &= -\frac{\partial \mathbf{B}}{\partial t} \\ + \nabla \times \mathbf{B} &= \mu_0 \mathbf{J} + \mu_0 \varepsilon_0 \frac{\partial \mathbf{E}}{\partial t}. + +The fluids are coupled to the fields through, + +.. math:: + + \rho &= \rho_{ptcl}+\sum_s q_sN_s \\ + \mathbf{J} &= \mathbf{J}_{ptcl}+\sum_s q_sN_s\mathbf{V}_s \\ + \mathbf{V}_s &= \frac{ \mathbf{U}_s }{ \sqrt{ 1 + \mathbf{U}_s^2/c^2} } \\ + (N\mathbf{U})_s &= N_s\mathbf{U}_s + +where the particle quantities are calculated by the PIC algorithm. + + +Implementation details +---------------------- + +The fluid timeloop is embedded inside the standard PIC timeloop and consists of +the following steps: 1. Higuera and Cary push of the momentum 2. Non-inertial (momentum source) +terms (only in cylindrical geometry) 3. boundary conditions and MPI Communications 4. MUSCL +scheme for advection terms 5. Current and Charge Deposition. The figure here gives +a visual representation of these steps, and we describe each of these in more detail. + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 + :alt: Fluid Loop embedded within the overall PIC loop. + +Step 0: **Preparation** + Before the fluid loop begins, it is assumed that the program is in the state where fields :math:`\mathbf{E}` + and :math:`\mathbf{B}` are available integer timestep. The + fluids themselves are represented by arrays of fluid quantities (density and + momentum density, :math:`\mathbf{Q} \equiv \{ N, NU_x, NU_y, NU_z \}`) known + on a nodal grid and at half-integer timestep. + +Step 1: **Higuera and Cary Push** + The time staggering of the fields is used by the momentum source term, which is solved with a the + Higeura and Cary push (Higuera et al, 2017). We do not adopt spatial + grid staggering, all discretized fluid quantities exist on the nodal grid. External fields + can be included at this step. + +Step 2: **Non-inertial Terms** + In RZ, the divergence of the flux terms has additional non-zero elements outside of the + derivatives. These terms are Strang split and are time integrated via equation 2.18 from (Osher et al, 1988), + which is the SSP-RK3 integrator. + +Step 3: **Boundary Conditions and Communications** + At this point, the code applies boundary conditions (assuming Neumann boundary conditions + for the fluid quantities) and exchanges guard cells between + MPI ranks in preparation of derivative terms in the next step. + +Step 4: **Advective Push** + For the advective term, a MUSCL scheme with a low-diffusion minmod slope + limiting is used. We further simplify the conservative equations in terms of primitive + variables, :math:`\{ N, U_x, U_y, U_z \}`. Which we found to be + more stable than conservative variables for the MUSCL reconstruction. Details of + the scheme can be found here (Van Leer et al, 1984). + +Step 5: **Current and Charge Deposition** + Once this series of steps is complete and the fluids have been evolved by an entire + timestep, the current and charge is deposited onto the grid and added to the total current and charge + densities. + +.. note:: + The algorithm is safe with zero fluid density. + + It also implements a positivity limiter on the density to prevent negative density regions from forming. + + There is currently no ability to perform azimuthal mode decomposition in RZ. + + Mesh refinement is not supported for the fluids. + + The implemented MUSCL scheme has a simplifed slope averaging, see the extended writeup for details. + + More details on the precise implementation will be made available online soon. + +.. warning:: + If using the fluid model with the Kinetic-Fluid Hybrid model or the electrostatic solver, there is a known + issue that the fluids deposit at a half-timestep offset in the charge-density. + +.. raw:: html + +
+ +.. raw:: html + +
+ +Higuera, Adam V., and John R. Cary. "Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields." Physics of Plasmas 24.5 (2017). + +.. raw:: html + +
+ +.. raw:: html + +
+ +Osher, Stanley, and Chi-Wang Shu. "Efficient implementation of essentially non-oscillatory shock-capturing schemes." J. Comput. Phys 77.2 (1988): 439-471. + + +.. raw:: html + +
+ +.. raw:: html + +
+ +Van Leer, Bram. "On the relation between the upwind-differencing schemes of Godunov, Engquist–Osher and Roe." SIAM Journal on Scientific and statistical Computing 5.1 (1984): 1-20. diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4514e6f3fb0..17504a8065f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1136,6 +1136,18 @@ Particle initialization Resampling is performed everytime the number of macroparticles per cell of the species averaged over the whole simulation domain exceeds this parameter. + +.. _running-cpp-parameters-fluids: + +Cold Relativistic Fluid initialization +-------------------------------------- + +* ``fluids.species_names`` (`strings`, separated by spaces) + Defines the names of each fluid species. It is a required input to create and evolve fluid species using the cold relativistic fluid equations. + Most of the parameters described in the section "Particle initialization" can also be used to initialize fluid properties (e.g. initial density distribution). + For fluid-specific inputs we use `` as a placeholder. Also see external fields + for how to specify these for fluids as the function names differ. + .. _running-cpp-parameters-laser: Laser initialization @@ -1499,6 +1511,37 @@ Applied to Particles and :math:`E_z = 0`, and :math:`B_x = \mathrm{strength} \cdot y`, :math:`B_y = -\mathrm{strength} \cdot x`, and :math:`B_z = 0`. + +Applied to Cold Relativistic Fluids +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``.E_ext_init_style`` & ``.B_ext_init_style`` (string) optional (default "none") + These parameters determine the type of the external electric and + magnetic fields respectively that are applied directly to the cold relativistic fluids at every timestep. + The field values are specified in the lab frame. + With the default ``none`` style, no field is applied. + Possible values are ``parse_E_ext_function`` or ``parse_B_ext_function``. + + * ``parse_E_ext_function`` or ``parse_B_ext_function``: the field is specified as an analytic + expression that is a function of space (x,y,z) and time (t), relative to the lab frame. + The E-field is specified by the input parameters: + + * ``.Ex_external_function(x,y,z,t)`` + + * ``.Ey_external_function(x,y,z,t)`` + + * ``.Ez_external_function(x,y,z,t)`` + + The B-field is specified by the input parameters: + + * ``.Bx_external_function(x,y,z,t)`` + + * ``.By_external_function(x,y,z,t)`` + + * ``.Bz_external_function(x,y,z,t)`` + + Note that the position is defined in Cartesian coordinates, as a function of (x,y,z), even for RZ. + Accelerator Lattice ^^^^^^^^^^^^^^^^^^^ @@ -2988,6 +3031,57 @@ Reduced Diagnostics 1 or 0, it is possible to compute the charge on only some part of the embedded boundary. + * ``ColliderRelevant`` + This diagnostics computes properties of two colliding beams that are relevant for particle colliders. + Two species must be specified. Photon species are not supported yet. + It is assumed that the two species propagate and collide along the ``z`` direction. + The output columns (for 3D-XYZ) are the following, where the minimum, average and maximum + are done over the whole species: + + [0]: simulation step (iteration). + + [1]: time (s). + + [2]: time derivative of the luminosity (:math:`m^{-2}s^{-1}`) defined as: + + .. math:: + + \frac{dL}{dt} = 2 c \iiint n_1(x,y,z) n_2(x,y,z) dx dy dz + + where :math:`n_1`, :math:`n_2` are the number densities of the two colliding species. + + [3], [4], [5]: If, QED is enabled, the minimum, average and maximum values of the quantum parameter :math:`\chi` of species 1: + :math:`\chi_{min}`, + :math:`\langle \chi \rangle`, + :math:`\chi_{max}`. + If QED is not enabled, these numbers are not computed. + + [6], [7]: The average and standard deviation of the values of the transverse coordinate :math:`x` (m) of species 1: + :math:`\langle x \rangle`, + :math:`\sqrt{\langle x- \langle x \rangle \rangle^2}`. + + [8], [9]: The average and standard deviation of the values of the transverse coordinate :math:`y` (m) of species 1: + :math:`\langle y \rangle`, + :math:`\sqrt{\langle y- \langle y \rangle \rangle^2}`. + + [10], [11], [12], [13]: The minimum, average, maximum and standard deviation of the angle :math:`\theta_x = \angle (u_x, u_z)` (rad) of species 1: + :math:`{\theta_x}_{min}`, + :math:`\langle \theta_x \rangle`, + :math:`{\theta_x}_{max}`, + :math:`\sqrt{\langle \theta_x- \langle \theta_x \rangle \rangle^2}`. + + [14], [15], [16], [17]: The minimum, average, maximum and standard deviation of the angle :math:`\theta_y = \angle (u_y, u_z)` (rad) of species 1: + :math:`{\theta_y}_{min}`, + :math:`\langle \theta_y \rangle`, + :math:`{\theta_y}_{max}`, + :math:`\sqrt{\langle \theta_y- \langle \theta_y \rangle \rangle^2}`. + + [18], ..., [32]: Analogous quantities for species 2. + + For 2D-XZ, :math:`y`-related quantities are not outputted. + For 1D-Z, :math:`x`-related and :math:`y`-related quantities are not outputted. + RZ geometry is not supported yet. + * ``.intervals`` (`string`) Using the `Intervals Parser`_ syntax, this string defines the timesteps at which reduced diagnostics are written to file. diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py new file mode 100755 index 00000000000..a376ed872a9 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2023 Grant Johnson, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: +# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 +import os +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +n0 = 20.e23 +# Plasma frequency +wp = np.sqrt((n0*e**2)/(m_e*epsilon_0)) +kp = wp/c +tau = 15.e-15 +a0 = 2.491668 +e = -e #Electrons +lambda_laser = 0.8e-6 + +zmin = -20e-6; zmax = 100.e-6; Nz = 10240 + +# Compute the theory + +# Call the ode solver +from scipy.integrate import odeint + + +# ODE Function +def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): + phi1, phi2 = phi + a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2))*np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 + dphi1_dxi = phi2 + dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1)**2) - 0.5) + return [dphi1_dxi, dphi2_dxi] + +# Call odeint to solve the ODE +xi_span = [-20e-6, 100e-6] +xi_0 = 0e-6 +phi0 = [0.0, 0.0] +dxi = (zmax-zmin)/Nz +xi = zmin + dxi*( 0.5 + np.arange(Nz) ) +phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) + +# Change array direction to match the simulations +xi = -xi[::-1] +phi = phi[::-1] +xi_0 = -0e-6 +phi2 = phi[:, 0] +Ez = -phi[:, 1] + +# Compute the derived quantities +a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2)) *np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 +gamma_perp_sq = 1 + a_sq +n = n0 * (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)**2) +uz = (gamma_perp_sq - (1 + phi2)**2) / (2 * (1 + phi2)) +gamma = (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)) + +# Theory Components [convert to si] +uz *= c +J_th = np.multiply( np.divide(uz,gamma), n ) +J_th *= e +rho_th = e*n +E_th = Ez +E_th *= ((m_e*c*c)/e) +V_th = np.divide(uz,gamma) +V_th /= c +# Remove the ions +rho_th = rho_th - e*n0 + +# Dicate which region to compare solutions over +# (Currently this is the full domain) +min_i = 0 +max_i = 10240 + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) +# Check the validity of the fields +error_rel = 0 +for field in ['Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim[min_i:max_i]-E_th[min_i:max_i]).max()/abs(E_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim[min_i:max_i]-J_th[min_i:max_i]).max()/abs(J_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] + #rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim[min_i:max_i]-rho_th[min_i:max_i]).max()/abs(rho_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +V_sim = np.divide(J_sim,rho_sim) +V_sim /= c + +# Create a figure with 2 rows and 2 columns +fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) + +# Titles and labels +titles = ['Ez', 'rho', 'Jz', 'Vz/c'] +xlabel = r'Xi' +ylabel = ['Ez', 'rho', 'Jz', 'Vz/c'] + +# Plotting loop +for i in range(3): + ax = axes[i // 2, i % 2] # Get the current subplot + + # Plot theoretical data + ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label='Theoretical') + + # Plot simulated data + ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label='Simulated') + + # Set titles and labels + ax.set_title(f'{titles[i]} vs Xi') + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel[i]) + + # Add legend + ax.legend() + +# Adjust subplot layout +plt.tight_layout() + +# Save the figure +plt.savefig('wfa_fluid_nonlinear_1d_analysis.png') + +plt.show() + + +tolerance_rel = 0.20 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py new file mode 100755 index 00000000000..4f4ae2a812a --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2023 Grant Johnson, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: +# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 +import os +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +n0 = 20.e23 +# Plasma frequency +wp = np.sqrt((n0*e**2)/(m_e*epsilon_0)) +kp = wp/c +tau = 15.e-15 +a0 = 2.491668 +e = -e #Electrons +lambda_laser = 0.8e-6 + +zmin = -20e-6; zmax = 100.e-6; Nz = 4864 + +# Compute the theory + +# Call the ode solver +from scipy.integrate import odeint + + +# ODE Function +def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): + phi1, phi2 = phi + a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2))*np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 + dphi1_dxi = phi2 + dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1)**2) - 0.5) + return [dphi1_dxi, dphi2_dxi] + +# Call odeint to solve the ODE +xi_span = [-20e-6, 100e-6] +xi_0 = 0e-6 +phi0 = [0.0, 0.0] +dxi = (zmax-zmin)/Nz +xi = zmin + dxi*( 0.5 + np.arange(Nz) ) +phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) + +# Change array direction to match the simulations +xi = -xi[::-1] +phi = phi[::-1] +xi_0 = -0e-6 +phi2 = phi[:, 0] +Ez = -phi[:, 1] + +# Compute the derived quantities +a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2)) *np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 +gamma_perp_sq = 1 + a_sq +n = n0 * (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)**2) +uz = (gamma_perp_sq - (1 + phi2)**2) / (2 * (1 + phi2)) +gamma = (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)) + +# Theory Components [convert to si] +uz *= c +J_th = np.multiply( np.divide(uz,gamma), n ) +J_th *= e +rho_th = e*n +E_th = Ez +E_th *= ((m_e*c*c)/e) +V_th = np.divide(uz,gamma) +V_th /= c +# Remove the ions +rho_th = rho_th - e*n0 + +# Dicate which region to compare solutions over (cuttoff 0's from BTD extra) +min_i = 200 +max_i = 4864 + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) +# Check the validity of the fields +error_rel = 0 +for field in ['Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim[min_i:max_i]-E_th[min_i:max_i]).max()/abs(E_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim[min_i:max_i]-J_th[min_i:max_i]).max()/abs(J_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] + #rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim[min_i:max_i]-rho_th[min_i:max_i]).max()/abs(rho_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +V_sim = np.divide(J_sim,rho_sim) +V_sim /= c + +# Create a figure with 2 rows and 2 columns +fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) + +# Titles and labels +titles = ['Ez', 'rho', 'Jz', 'Vz/c'] +xlabel = r'Xi' +ylabel = ['Ez', 'rho', 'Jz', 'Vz/c'] + +# Plotting loop +for i in range(3): + ax = axes[i // 2, i % 2] # Get the current subplot + + # Plot theoretical data + ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label='Theoretical') + + # Plot simulated data + ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label='Simulated') + + # Set titles and labels + ax.set_title(f'{titles[i]} vs Xi') + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel[i]) + + # Add legend + ax.legend() + +# Adjust subplot layout +plt.tight_layout() + +# Save the figure +plt.savefig('wfa_fluid_nonlinear_1d_analysis.png') + +plt.show() + + +tolerance_rel = 0.30 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids new file mode 100644 index 00000000000..73fa6b7283f --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids @@ -0,0 +1,72 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 40000 +amr.n_cell = 10240 +amr.max_grid_size = 512 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 512 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 1 +geometry.prob_lo = -120.e-6 # physical domain +geometry.prob_hi = 0.e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = pec +boundary.field_hi = pec + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 0 +warpx.cfl = 0.45 #Fluid CFL < 0.5 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # units of speed of light +warpx.do_dynamic_scheduling = 0 +warpx.serialize_initial_conditions = 1 + +################################# +############ PLASMA ############# +################################# +fluids.species_names = electrons ions + +electrons.species_type = electron +electrons.profile = parse_density_function +electrons.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +electrons.momentum_distribution_type = "at_rest" + +ions.charge = q_e +ions.mass = m_p +ions.profile = parse_density_function +ions.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +ions.momentum_distribution_type = "at_rest" + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ LASER ############## +################################# +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -11.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 0. 1. 0. # The main polarization vector +laser1.e_max = 10.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 5.e-6 # The waist of the laser (in m) +laser1.profile_duration = 15.e-15 # The duration of the laser (in s) +laser1.profile_t_peak = 30.e-15 # Time at which the laser reaches its peak (in s) +laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in m) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in m) + +# Diagnostics +diagnostics.diags_names = diag1 + +# LAB +diag1.intervals = 20000 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted new file mode 100644 index 00000000000..f9437716f66 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted @@ -0,0 +1,79 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 10000 +amr.n_cell = 5120 +amr.max_grid_size = 1024 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 1024 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 1 +geometry.prob_lo = -120.e-6 # physical domain +geometry.prob_hi = 0.e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = pec +boundary.field_hi = pec + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 0 +warpx.cfl = 0.45 #Fluid CFL < 0.5 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # units of speed of light +warpx.do_dynamic_scheduling = 0 +warpx.serialize_initial_conditions = 1 + +### BOOST ### +warpx.gamma_boost = 1.5 +warpx.boost_direction = z + +################################# +############ PLASMA ############# +################################# +fluids.species_names = electrons ions + +electrons.species_type = electron +electrons.profile = parse_density_function +electrons.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +electrons.momentum_distribution_type = "at_rest" + +ions.charge = q_e +ions.mass = m_p +ions.profile = parse_density_function +ions.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +ions.momentum_distribution_type = "at_rest" + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ LASER ############## +################################# +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -11.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 0. 1. 0. # The main polarization vector +laser1.e_max = 10.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 5.e-6 # The waist of the laser (in m) +laser1.profile_duration = 15.e-15 # The duration of the laser (in s) +laser1.profile_t_peak = 30.e-15 # Time at which the laser reaches its peak (in s) +laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in m) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in m) + +# Diagnostics +diagnostics.diags_names = diag1 + + +# BOOST +diag1.diag_type = BackTransformed +diag1.do_back_transformed_fields = 1 +diag1.num_snapshots_lab = 2 +diag1.dt_snapshots_lab = 6.e-13 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py b/Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py new file mode 100755 index 00000000000..b23bb69d52c --- /dev/null +++ b/Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +import os +import sys + +import numpy as np +import openpmd_api as io +import pandas as pd +from scipy.constants import c, e, hbar, m_e + +sys.path.append('../../../../warpx/Regression/Checksum/') +import checksumAPI + +sys.path.append('../../../../warpx/Tools/Parser/') +from input_file_parser import parse_input_file + +E_crit = m_e**2*c**3/(e*hbar) +B_crit = m_e**2*c**2/(e*hbar) + +def chi(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz): + gamma = np.sqrt(1.+ux**2+uy**2+uz**2) + vx = ux / gamma * c + vy = uy / gamma * c + vz = uz / gamma * c + tmp1x = Ex + vy*Bz - vz*By + tmp1y = Ey - vx*Bz + vz*Bx + tmp1z = Ez + vx*By - vy*Bx + tmp2 = (Ex*vx + Ey*vy + Ez*vz)/c + chi = gamma/E_crit*np.sqrt(tmp1x**2+tmp1y**2+tmp1z**2 - tmp2**2) + return chi + +def dL_dt(): + series = io.Series("diags/diag2/openpmd_%T.h5",io.Access.read_only) + iterations = np.asarray(series.iterations) + lumi = [] + for n,ts in enumerate(iterations): + it = series.iterations[ts] + rho1 = it.meshes["rho_beam_e"] + dV = np.prod(rho1.grid_spacing) + rho1 = it.meshes["rho_beam_e"][io.Mesh_Record_Component.SCALAR].load_chunk() + rho2 = it.meshes["rho_beam_p"][io.Mesh_Record_Component.SCALAR].load_chunk() + beam_e_charge = it.particles["beam_e"]["charge"][io.Mesh_Record_Component.SCALAR].load_chunk() + beam_p_charge = it.particles["beam_p"]["charge"][io.Mesh_Record_Component.SCALAR].load_chunk() + q1 = beam_e_charge[0] + if not np.all(beam_e_charge == q1): + sys.exit('beam_e particles do not have the same charge') + q2 = beam_p_charge[0] + if not np.all(beam_p_charge == q2): + sys.exit('beam_p particles do not have the same charge') + series.flush() + n1 = rho1/q1 + n2 = rho2/q2 + l = 2*np.sum(n1*n2)*dV*c + lumi.append(l) + return lumi + +input_dict = parse_input_file('inputs_3d_multiple_particles') +Ex, Ey, Ez = [float(w) for w in input_dict['particles.E_external_particle']] +Bx, By, Bz = [float(w) for w in input_dict['particles.B_external_particle']] + +CollDiagFname='diags/reducedfiles/ColliderRelevant_beam_e_beam_p.txt' +df = pd.read_csv(CollDiagFname, sep=" ", header=0) + +for species in ['beam_p', 'beam_e']: + + ux1, ux2, ux3 = [float(w) for w in input_dict[f'{species}.multiple_particles_ux']] + uy1, uy2, uy3 = [float(w) for w in input_dict[f'{species}.multiple_particles_uy']] + uz1, uz2, uz3 = [float(w) for w in input_dict[f'{species}.multiple_particles_uz']] + + x = np.array([float(w) for w in input_dict[f'{species}.multiple_particles_pos_x']]) + y = np.array([float(w) for w in input_dict[f'{species}.multiple_particles_pos_y']]) + + w = np.array([float(w) for w in input_dict[f'{species}.multiple_particles_weight']]) + + CHI_ANALYTICAL = np.array([chi(ux1, uy1, uz1, Ex, Ey, Ez, Bx, By, Bz), + chi(ux2, uy2, uz2, Ex, Ey, Ez, Bx, By, Bz), + chi(ux3, uy3, uz3, Ex, Ey, Ez, Bx, By, Bz)]) + THETAX = np.array([np.arctan2(ux1, uz1), np.arctan2(ux2, uz2), np.arctan2(ux3, uz3)]) + THETAY = np.array([np.arctan2(uy1, uz1), np.arctan2(uy2, uz2), np.arctan2(uy3, uz3)]) + + # CHI MAX + fname=f'diags/reducedfiles/ParticleExtrema_{species}.txt' + chimax_pe = np.loadtxt(fname)[:,19] + chimax_cr = df[[col for col in df.columns if f'chi_max_{species}' in col]].to_numpy() + assert np.allclose(np.max(CHI_ANALYTICAL), chimax_cr, rtol=1e-8) + assert np.allclose(chimax_pe, chimax_cr, rtol=1e-8) + + # CHI MIN + fname=f'diags/reducedfiles/ParticleExtrema_{species}.txt' + chimin_pe = np.loadtxt(fname)[:,18] + chimin_cr = df[[col for col in df.columns if f'chi_min_{species}' in col]].to_numpy() + assert np.allclose(np.min(CHI_ANALYTICAL), chimin_cr, rtol=1e-8) + assert np.allclose(chimin_pe, chimin_cr, rtol=1e-8) + + # CHI AVERAGE + chiave_cr = df[[col for col in df.columns if f'chi_ave_{species}' in col]].to_numpy() + assert np.allclose(np.average(CHI_ANALYTICAL, weights=w), chiave_cr, rtol=1e-8) + + # X AVE STD + x_ave_cr = df[[col for col in df.columns if f']x_ave_{species}' in col]].to_numpy() + x_std_cr = df[[col for col in df.columns if f']x_std_{species}' in col]].to_numpy() + x_ave = np.average(x, weights=w) + x_std = np.sqrt(np.average((x-x_ave)**2, weights=w)) + assert np.allclose(x_ave, x_ave_cr, rtol=1e-8) + assert np.allclose(x_std, x_std_cr, rtol=1e-8) + + # Y AVE STD + y_ave_cr = df[[col for col in df.columns if f']y_ave_{species}' in col]].to_numpy() + y_std_cr = df[[col for col in df.columns if f']y_std_{species}' in col]].to_numpy() + y_ave = np.average(y, weights=w) + y_std = np.sqrt(np.average((y-y_ave)**2, weights=w)) + assert np.allclose(y_ave, y_ave_cr, rtol=1e-8) + assert np.allclose(y_std, y_std_cr, rtol=1e-8) + + # THETA X MIN AVE MAX STD + thetax_min_cr = df[[col for col in df.columns if f'theta_x_min_{species}' in col]].to_numpy() + thetax_ave_cr = df[[col for col in df.columns if f'theta_x_ave_{species}' in col]].to_numpy() + thetax_max_cr = df[[col for col in df.columns if f'theta_x_max_{species}' in col]].to_numpy() + thetax_std_cr = df[[col for col in df.columns if f'theta_x_std_{species}' in col]].to_numpy() + thetax_min = np.min(THETAX) + thetax_ave = np.average(THETAX, weights=w) + thetax_max = np.max(THETAX) + thetax_std = np.sqrt(np.average((THETAX-thetax_ave)**2, weights=w)) + assert np.allclose(thetax_min, thetax_min_cr, rtol=1e-8) + assert np.allclose(thetax_ave, thetax_ave_cr, rtol=1e-8) + assert np.allclose(thetax_max, thetax_max_cr, rtol=1e-8) + assert np.allclose(thetax_std, thetax_std_cr, rtol=1e-8) + + # THETA Y MIN AVE MAX STD + thetay_min_cr = df[[col for col in df.columns if f'theta_y_min_{species}' in col]].to_numpy() + thetay_ave_cr = df[[col for col in df.columns if f'theta_y_ave_{species}' in col]].to_numpy() + thetay_max_cr = df[[col for col in df.columns if f'theta_y_max_{species}' in col]].to_numpy() + thetay_std_cr = df[[col for col in df.columns if f'theta_y_std_{species}' in col]].to_numpy() + thetay_min = np.min(THETAY) + thetay_ave = np.average(THETAY, weights=w) + thetay_max = np.max(THETAY) + thetay_std = np.sqrt(np.average((THETAY-thetay_ave)**2, weights=w)) + assert np.allclose(thetay_min, thetay_min_cr, rtol=1e-8) + assert np.allclose(thetay_ave, thetay_ave_cr, rtol=1e-8) + assert np.allclose(thetay_max, thetay_max_cr, rtol=1e-8) + assert np.allclose(thetay_std, thetay_std_cr, rtol=1e-8) + + # dL/dt + dL_dt_cr = df[[col for col in df.columns if 'dL_dt' in col]].to_numpy() + assert np.allclose(dL_dt_cr, dL_dt(), rtol=1e-8) + +# Checksum analysis +plotfile = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, plotfile) diff --git a/Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles b/Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles new file mode 100644 index 00000000000..1efc68c33b0 --- /dev/null +++ b/Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles @@ -0,0 +1,130 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.nx = 8 +my_constants.ny = 8 +my_constants.nz = 8 + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 1 +amr.n_cell = nx ny nz +amr.max_grid_size = 4 +amr.blocking_factor = 4 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 8 8 8 +particles.do_tiling = 0 +warpx.use_filter = 0 + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic +boundary.particle_lo = periodic periodic periodic +boundary.particle_hi = periodic periodic periodic + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = ckc +warpx.cfl = 0.99 +algo.particle_shape = 1 + +################################# +############ FIELDS ############# +################################# +particles.E_ext_particle_init_style = constant +particles.B_ext_particle_init_style = constant +particles.E_external_particle = 10000. 0. 0. +particles.B_external_particle = 0. 5000. 0. + +################################# +########### PARTICLES ########### +################################# +particles.species_names = pho beam_p beam_e +particles.photon_species = pho + +beam_e.species_type = electron +beam_e.injection_style = MultipleParticles +beam_e.multiple_particles_pos_x = 4.5 3.5 0.5 +beam_e.multiple_particles_pos_y = 4.5 2.5 1.5 +beam_e.multiple_particles_pos_z = 4.5 1.5 1.5 +beam_e.multiple_particles_ux = 0.3 0.2 0.1 +beam_e.multiple_particles_uy = 0.4 -0.3 -0.1 +beam_e.multiple_particles_uz = 0.3 0.1 -10. +beam_e.multiple_particles_weight = 1. 2 3 +beam_e.initialize_self_fields = 0 +beam_e.self_fields_required_precision = 5e-10 +beam_e.do_qed_quantum_sync = 1 +beam_e.qed_quantum_sync_phot_product_species = pho +beam_e.do_not_push = 1 +beam_e.do_not_deposit = 1 + +beam_p.species_type = positron +beam_p.injection_style = MultipleParticles +beam_p.multiple_particles_pos_x = 4.5 3.5 0.5 +beam_p.multiple_particles_pos_y = 4.5 2.5 1.5 +beam_p.multiple_particles_pos_z = 4.5 1.5 1.5 +beam_p.multiple_particles_ux = 0.3 0.2 0.1 +beam_p.multiple_particles_uy = 0.4 -0.3 -0.1 +beam_p.multiple_particles_uz = 0.3 0.1 -10. +beam_p.multiple_particles_weight = 1. 2 3 +beam_p.initialize_self_fields = 0 +beam_p.self_fields_required_precision = 5e-10 +beam_p.do_qed_quantum_sync = 1 +beam_p.qed_quantum_sync_phot_product_species = pho +beam_p.do_not_push = 1 +beam_p.do_not_deposit = 1 + +pho.species_type = photon +pho.injection_style = none + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 +warpx.do_qed_schwinger = 0 + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho_beam_e rho_beam_p rho +diag1.species = pho beam_e beam_p +diag1.format = plotfile +#diag1.dump_last_timestep = 1 + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.write_species = 1 +diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho_beam_e rho_beam_p rho +diag2.species = pho beam_e beam_p +diag2.format = openpmd +diag2.openpmd_backend = h5 +#diag2.dump_last_timestep = 1 + +# REDUCED +warpx.reduced_diags_names = ParticleExtrema_beam_e ParticleExtrema_beam_p ColliderRelevant_beam_e_beam_p + +ColliderRelevant_beam_e_beam_p.type = ColliderRelevant +ColliderRelevant_beam_e_beam_p.intervals = 1 +ColliderRelevant_beam_e_beam_p.species =beam_e beam_p + +ParticleExtrema_beam_e.type = ParticleExtrema +ParticleExtrema_beam_e.intervals = 1 +ParticleExtrema_beam_e.species = beam_e + +ParticleExtrema_beam_p.type = ParticleExtrema +ParticleExtrema_beam_p.intervals = 1 +ParticleExtrema_beam_p.species = beam_p diff --git a/Examples/Tests/langmuir_fluids/analysis_1d.py b/Examples/Tests/langmuir_fluids/analysis_1d.py new file mode 100755 index 00000000000..2d1a8f69d1d --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_1d.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2022 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Prabhat Kumar, Axel Huebl, Grant Johnson +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rt`. This simulates a 1D periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\sin(k_z z)\sin( \omega_p t)$$ +import os +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +epsilon = 0.01 +n = 4.e24 +n_osc_z = 2 +zmin = -20e-6; zmax = 20.e-6; Nz = 128 + +# Wave vector of the wave +kz = 2.*np.pi*n_osc_z/(zmax-zmin) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) + +k = {'Ez':kz,'Jz':kz} +cos = {'Ez':(1,1,0), 'Jz':(1,1,0)} +cos_rho = {'rho': (1,1,1)} + +def get_contribution( is_cos, k ): + du = (zmax-zmin)/Nz + u = zmin + du*( 0.5 + np.arange(Nz) ) + if is_cos == 1: + return( np.cos(k*u) ) + else: + return( np.sin(k*u) ) + +def get_theoretical_field( field, t ): + amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + cos_flag = cos[field] + z_contribution = get_contribution( cos_flag[2], kz ) + + E = amplitude * z_contribution + + return( E ) + +def get_theoretical_J_field( field, t ): + # wpdt/2 accounts for the Yee halfstep offset of the current + dt = t / 80 # SPECIFIC to config parameters! + amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + cos_flag = cos[field] + + z_contribution = get_contribution( cos_flag[2], kz ) + + J = amplitude * z_contribution + + return( J ) + +def get_theoretical_rho_field( field, t ): + amplitude = epsilon_0 * epsilon * (m_e*c**2*(kz*kz))/e * np.sin(wp*t) + cos_flag = cos_rho[field] + z_contribution = get_contribution( cos_flag[2], kz) + + rho = amplitude * z_contribution + + return( rho ) + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) +# Check the validity of the fields +error_rel = 0 +for field in ['Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,0,0] + E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim-E_th).max()/abs(E_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,0,0] + J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim-J_th).max()/abs(J_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] + rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Plot the last field from the loop (Ez at iteration 80) +plt.subplot2grid( (1,2), (0,0) ) +plt.plot( E_sim ) +#plt.colorbar() +plt.title('Ez, last iteration\n(simulation)') +plt.subplot2grid( (1,2), (0,1) ) +plt.plot( E_th ) +#plt.colorbar() +plt.title('Ez, last iteration\n(theory)') +plt.tight_layout() +plt.savefig('langmuir_fluid_multi_1d_analysis.png') + +tolerance_rel = 0.05 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/analysis_2d.py b/Examples/Tests/langmuir_fluids/analysis_2d.py new file mode 100755 index 00000000000..cf5d2fb44de --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_2d.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Grant Johnson +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rt`. This simulates a 3D periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ +import os +import sys + +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +epsilon = 0.01 +n = 4.e24 +n_osc_x = 2 +n_osc_z = 2 +xmin = -20e-6; xmax = 20.e-6; Nx = 128 +zmin = -20e-6; zmax = 20.e-6; Nz = 128 + +# Wave vector of the wave +kx = 2.*np.pi*n_osc_x/(xmax-xmin) +kz = 2.*np.pi*n_osc_z/(zmax-zmin) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) + +k = {'Ex':kx, 'Ez':kz, 'Jx':kx, 'Jz':kz} +cos = {'Ex': (0,1,1), 'Ez':(1,1,0),'Jx': (0,1,1), 'Jz':(1,1,0)} +cos_rho = {'rho': (1,1,1)} + +def get_contribution( is_cos, k ): + du = (xmax-xmin)/Nx + u = xmin + du*( 0.5 + np.arange(Nx) ) + if is_cos == 1: + return( np.cos(k*u) ) + else: + return( np.sin(k*u) ) + +def get_theoretical_field( field, t ): + amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag[0], kx ) + z_contribution = get_contribution( cos_flag[2], kz ) + + E = amplitude * x_contribution[:, np.newaxis ] \ + * z_contribution[np.newaxis, :] + + return( E ) + +def get_theoretical_J_field( field, t ): + # wpdt/2 accounts for the Yee halfstep offset of the current + dt = t / 40 # SPECIFIC to config parameters! + amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag[0], kx ) + z_contribution = get_contribution( cos_flag[2], kz ) + + J = amplitude * x_contribution[:, np.newaxis] \ + * z_contribution[np.newaxis, :] + + return( J ) + +def get_theoretical_rho_field( field, t ): + amplitude = epsilon_0 * epsilon * (m_e*c**2*(kx*kx+kz*kz))/e * np.sin(wp*t) + cos_flag = cos_rho[field] + x_contribution = get_contribution( cos_flag[0], kx ) + z_contribution = get_contribution( cos_flag[2], kz ) + + rho = amplitude * x_contribution[:, np.newaxis] \ + * z_contribution[ np.newaxis, :] + + return( rho ) + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) +edge = np.array([(ds.domain_left_edge[1]).item(), (ds.domain_right_edge[1]).item(), \ + (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) + +# Check the validity of the fields +error_rel = 0 +for field in ['Ex', 'Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,:,0] + E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim-E_th).max()/abs(E_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jx', 'Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,:,0] + J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim-J_th).max()/abs(J_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,:,0] + rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Plot the last field from the loop (Ez at iteration 40) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +# First plot +vmin = E_sim.min() +vmax = E_sim.max() +cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') +im1 = ax1.imshow(E_sim, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb1 = fig.colorbar(im1, cax = cax1) +ax1.set_xlabel(r'$z$') +ax1.set_ylabel(r'$x$') +ax1.set_title(r'$E_z$ (sim)') +# Second plot +vmin = E_th.min() +vmax = E_th.max() +cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') +im2 = ax2.imshow(E_th, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb2 = fig.colorbar(im2, cax = cax2) +ax2.set_xlabel(r'$z$') +ax2.set_ylabel(r'$x$') +ax2.set_title(r'$E_z$ (theory)') +# Save figure +fig.tight_layout() +fig.savefig('Langmuir_fluid_multi_2d_analysis.png', dpi = 200) + +tolerance_rel = 0.05 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/analysis_3d.py b/Examples/Tests/langmuir_fluids/analysis_3d.py new file mode 100755 index 00000000000..0211956859e --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_3d.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2022 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Axel Huebl, Grant Johnson +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rt`. This simulates a 3D periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ +import os +import re +import sys + +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +epsilon = 0.01 +n = 4.e24 +n_osc_x = 2 +n_osc_y = 2 +n_osc_z = 2 +lo = [-20.e-6, -20.e-6, -20.e-6] +hi = [ 20.e-6, 20.e-6, 20.e-6] +Ncell = [64, 64, 64] + +# Wave vector of the wave +kx = 2.*np.pi*n_osc_x/(hi[0]-lo[0]) +ky = 2.*np.pi*n_osc_y/(hi[1]-lo[1]) +kz = 2.*np.pi*n_osc_z/(hi[2]-lo[2]) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) + +k = {'Ex':kx, 'Ey':ky, 'Ez':kz, 'Jx':kx, 'Jy':ky, 'Jz':kz} +cos = {'Ex': (0,1,1), 'Ey':(1,0,1), 'Ez':(1,1,0),'Jx': (0,1,1), 'Jy':(1,0,1), 'Jz':(1,1,0)} +cos_rho = {'rho': (1,1,1)} + +def get_contribution( is_cos, k, idim ): + du = (hi[idim]-lo[idim])/Ncell[idim] + u = lo[idim] + du*( 0.5 + np.arange(Ncell[idim]) ) + if is_cos[idim] == 1: + return( np.cos(k*u) ) + else: + return( np.sin(k*u) ) + +def get_theoretical_field( field, t ): + amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag, kx, 0 ) + y_contribution = get_contribution( cos_flag, ky, 1 ) + z_contribution = get_contribution( cos_flag, kz, 2 ) + + E = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ + * y_contribution[np.newaxis, :, np.newaxis] \ + * z_contribution[np.newaxis, np.newaxis, :] + + return( E ) + +def get_theoretical_J_field( field, t ): + # wpdt/2 accounts for the Yee halfstep offset of the current + dt = t / 40 # SPECIFIC to config parameters! + amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag, kx, 0 ) + y_contribution = get_contribution( cos_flag, ky, 1 ) + z_contribution = get_contribution( cos_flag, kz, 2 ) + + J = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ + * y_contribution[np.newaxis, :, np.newaxis] \ + * z_contribution[np.newaxis, np.newaxis, :] + + return( J ) + +def get_theoretical_rho_field( field, t ): + amplitude = epsilon_0 * epsilon * (m_e*c**2*(kx*kx+ky*ky+kz*kz))/e * np.sin(wp*t) + cos_flag = cos_rho[field] + x_contribution = get_contribution( cos_flag, kx, 0 ) + y_contribution = get_contribution( cos_flag, ky, 1 ) + z_contribution = get_contribution( cos_flag, kz, 2 ) + + rho = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ + * y_contribution[np.newaxis, :, np.newaxis] \ + * z_contribution[np.newaxis, np.newaxis, :] + + return( rho ) + +# Read the file +ds = yt.load(fn) + + +t0 = ds.current_time.to_value() +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) +edge = np.array([(ds.domain_left_edge[2]).item(), (ds.domain_right_edge[2]).item(), \ + (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) + +# Check the validity of the fields +error_rel = 0 +for field in ['Ex', 'Ey', 'Ez']: + E_sim = data[('mesh',field)].to_ndarray() + E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim-E_th).max()/abs(E_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + + +# Check the validity of the currents +for field in ['Jx', 'Jy', 'Jz']: + J_sim = data[('mesh',field)].to_ndarray() + J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim-J_th).max()/abs(J_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray() + rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + + +# Plot the last field from the loop (Ez at iteration 40) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +# First plot (slice at y=0) +E_plot = E_sim[:,Ncell[1]//2+1,:] +vmin = E_plot.min() +vmax = E_plot.max() +cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') +im1 = ax1.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb1 = fig.colorbar(im1, cax = cax1) +ax1.set_xlabel(r'$z$') +ax1.set_ylabel(r'$x$') +ax1.set_title(r'$E_z$ (sim)') +# Second plot (slice at y=0) +E_plot = E_th[:,Ncell[1]//2+1,:] +vmin = E_plot.min() +vmax = E_plot.max() +cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') +im2 = ax2.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb2 = fig.colorbar(im2, cax = cax2) +ax2.set_xlabel(r'$z$') +ax2.set_ylabel(r'$x$') +ax2.set_title(r'$E_z$ (theory)') +# Save figure +fig.tight_layout() +fig.savefig('Langmuir_fluid_multi_analysis.png', dpi = 200) + +tolerance_rel = 5e-2 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] + +if re.search( 'single_precision', fn ): + checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) +else: + checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/analysis_rz.py b/Examples/Tests/langmuir_fluids/analysis_rz.py new file mode 100755 index 00000000000..108d054e75a --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_rz.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +# Copyright 2019 David Grote, Maxence Thevenet, Grant Johnson, Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rz.rt`. This simulates a RZ periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_r = -\partial_r \phi = \epsilon \,\frac{mc^2}{e}\frac{2\,r}{w_0^2} \exp\left(-\frac{r^2}{w_0^2}\right) \sin(k_0 z) \sin(\omega_p t) +# $$ E_z = -\partial_z \phi = - \epsilon \,\frac{mc^2}{e} k_0 \exp\left(-\frac{r^2}{w_0^2}\right) \cos(k_0 z) \sin(\omega_p t) +# Unrelated to the Langmuir waves, we also test the plotfile particle filter function in this +# analysis script. +import os +import re +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +test_name = os.path.split(os.getcwd())[1] + +# Parse test name and check if current correction (psatd.current_correction) is applied +current_correction = True if re.search('current_correction', fn) else False + +# Parameters (these parameters must match the parameters in `inputs.multi.rz.rt`) +epsilon = 0.01 +n = 2.e24 +w0 = 5.e-6 +n_osc_z = 2 +rmin = 0e-6; rmax = 20.e-6; Nr = 64 +zmin = -20e-6; zmax = 20.e-6; Nz = 128 + +# Wave vector of the wave +k0 = 2.*np.pi*n_osc_z/(zmax-zmin) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +kp = wp/c + +def Er( z, r, epsilon, k0, w0, wp, t) : + """ + Return the radial electric field as an array + of the same length as z and r, in the half-plane theta=0 + """ + Er_array = \ + epsilon * m_e*c**2/e * 2*r/w0**2 * \ + np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) + return( Er_array ) + +def Ez( z, r, epsilon, k0, w0, wp, t) : + """ + Return the longitudinal electric field as an array + of the same length as z and r, in the half-plane theta=0 + """ + Ez_array = \ + - epsilon * m_e*c**2/e * k0 * \ + np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) + return( Ez_array ) + +def Jr( z, r, epsilon, k0, w0, wp, t) : + """ + Return the radial current density as an array + of the same length as z and r, in the half-plane theta=0 + """ + dt = t / 80 # SPECIFIC to config parameters! + Jr_array = \ + - epsilon_0 * epsilon * m_e*c**2/e * 2*r/w0**2 * \ + np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.cos( wp*t -wp*dt/2) * wp #phase_error = wp*dt/2 + return( Jr_array ) + +def Jz( z, r, epsilon, k0, w0, wp, t) : + """ + Return the longitudinal current density as an array + of the same length as z and r, in the half-plane theta=0 + """ + dt = t / 80 # SPECIFIC to config parameters! + Jz_array = \ + epsilon_0 * epsilon * m_e*c**2/e * k0 * \ + np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.cos( wp*t -wp*dt/2) * wp #phase_error = wp*dt/2 + return( Jz_array ) + +def rho( z, r, epsilon, k0, w0, wp, t) : + """ + Return the charge density as an array + of the same length as z and r, in the half-plane theta=0 + """ + rho_array = \ + epsilon_0 * epsilon * m_e*c**2/e * np.sin( wp*t ) * np.sin( k0*z ) * np.exp( -r**2/w0**2 ) * \ + ((4.0/(w0**2))*(1 - (r**2)/(w0**2)) + k0**2) + return( rho_array ) + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) + +# Get cell centered coordinates +dr = (rmax - rmin)/Nr +dz = (zmax - zmin)/Nz +coords = np.indices([Nr, Nz],'d') +rr = rmin + (coords[0] + 0.5)*dr +zz = zmin + (coords[1] + 0.5)*dz + +# Check the validity of the fields +overall_max_error = 0 +Er_sim = data[('boxlib','Er')].to_ndarray()[:,:,0] +Er_th = Er(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Er_sim-Er_th).max()/abs(Er_th).max() +print('Er: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +Ez_sim = data[('boxlib','Ez')].to_ndarray()[:,:,0] +Ez_th = Ez(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Ez_sim-Ez_th).max()/abs(Ez_th).max() +print('Ez: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +Jr_sim = data[('boxlib','jr')].to_ndarray()[:,:,0] +Jr_th = Jr(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Jr_sim-Jr_th).max()/abs(Jr_th).max() +print('Jr: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +Jz_sim = data[('boxlib','jz')].to_ndarray()[:,:,0] +Jz_th = Jz(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Jz_sim-Jz_th).max()/abs(Jz_th).max() +print('Jz: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +rho_sim = data[('boxlib','rho')].to_ndarray()[:,:,0] +rho_th = rho(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() +print('rho: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +# Plot the last field from the loop (Ez at iteration 40) +plt.subplot2grid( (1,2), (0,0) ) +plt.imshow( Ez_sim ) +plt.colorbar() +plt.title('Ez, last iteration\n(simulation)') +plt.subplot2grid( (1,2), (0,1) ) +plt.imshow( Ez_th ) +plt.colorbar() +plt.title('Ez, last iteration\n(theory)') +plt.tight_layout() +plt.savefig(test_name+'_analysis.png') + +error_rel = overall_max_error + +tolerance_rel = 0.08 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/inputs_1d b/Examples/Tests/langmuir_fluids/inputs_1d new file mode 100644 index 00000000000..48fda36c61f --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_1d @@ -0,0 +1,72 @@ +# Maximum number of time steps +max_step = 80 + +# number of grid points +amr.n_cell = 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 1 +geometry.prob_lo = -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 + +# Boundary condition +boundary.field_lo = periodic +boundary.field_hi = periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +warpx.use_filter = 0 + +# CFL +warpx.cfl = 0.8 + +# Parameters for the plasma wave +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*pi/20.e-6 # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Particles +fluids.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 openpmd +diag1.intervals = 40 +diag1.diag_type = Full +diag1.fields_to_plot = Ez jz rho + +openpmd.intervals = 40 +openpmd.diag_type = Full +openpmd.format = openpmd diff --git a/Examples/Tests/langmuir_fluids/inputs_2d b/Examples/Tests/langmuir_fluids/inputs_2d new file mode 100644 index 00000000000..d75fa0f3393 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_2d @@ -0,0 +1,69 @@ +# Maximum number of time steps +max_step = 80 + +# number of grid points +amr.n_cell = 128 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -20.e-6 -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 20.e-6 + +# Boundary condition +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.field_gathering = energy-conserving +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Parameters for the plasma wave +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*pi/20.e-6 # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Fluids +fluids.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 40 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ez jx jz rho diff --git a/Examples/Tests/langmuir_fluids/inputs_3d b/Examples/Tests/langmuir_fluids/inputs_3d new file mode 100644 index 00000000000..f92ccf0ed60 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_3d @@ -0,0 +1,74 @@ +# Parameters for the plasma wave +my_constants.max_step = 40 +my_constants.lx = 40.e-6 # length of sides +my_constants.dx = 6.25e-07 # grid cell size +my_constants.nx = lx/dx # number of cells in each dimension +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*2.*pi/lx # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Maximum number of time steps +max_step = max_step + +# number of grid points +amr.n_cell = nx nx nx + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = nx nx nx + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -lx/2. -lx/2. -lx/2. # physical domain +geometry.prob_hi = lx/2. lx/2. lx/2. + +# Boundary condition +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Fluids +fluids.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = max_step +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho diff --git a/Examples/Tests/langmuir_fluids/inputs_rz b/Examples/Tests/langmuir_fluids/inputs_rz new file mode 100644 index 00000000000..2be427cbce5 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_rz @@ -0,0 +1,72 @@ +# Parameters for the plasma wave +my_constants.max_step = 80 +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron density, #/m^3 +my_constants.wp = sqrt(n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k0 = 2.*pi/20.e-6 # longitudianl perturbation wavenumber +my_constants.w0 = 5.e-6 # transverse perturbation length +# Note: kp is calculated in SI for a density of 2e24 +# k0 is calculated so as to have 2 periods within the 40e-6 wide box. + +# Maximum number of time steps +max_step = max_step + +# number of grid points +amr.n_cell = 64 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = RZ +geometry.prob_lo = 0.e-6 -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 20.e-6 +boundary.field_lo = none periodic +boundary.field_hi = none periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.field_gathering = energy-conserving +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Having this turned on makes for a more sensitive test +warpx.do_dive_cleaning = 1 + +# Fluids +fluids.species_names = electrons ions + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon/kp*2*x/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon/kp*2*y/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z)" +electrons.momentum_function_uz(x,y,z) = "-epsilon/kp*k0*exp(-(x**2+y**2)/w0**2)*cos(k0*z)" + + +ions.charge = q_e +ions.mass = m_p +ions.profile = constant +ions.density = n0 # number of ions per m^3 +ions.momentum_distribution_type = at_rest + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 80 +diag1.diag_type = Full + +diag1.fields_to_plot = jr jz Er Ez Bt rho diff --git a/Examples/Tests/pml/analysis_pml_psatd.py b/Examples/Tests/pml/analysis_pml_psatd.py index ca3d7cc3012..50d0b2ac1c1 100755 --- a/Examples/Tests/pml/analysis_pml_psatd.py +++ b/Examples/Tests/pml/analysis_pml_psatd.py @@ -8,6 +8,7 @@ # License: BSD-3-Clause-LBNL import os +import re import sys import numpy as np @@ -19,13 +20,18 @@ filename = sys.argv[1] -############################ -### INITIAL LASER ENERGY ### -############################ -energy_start = 7.282940107273505e-08 # electromagnetic energy at iteration 50 +galilean = True if re.search("galilean", filename) else False + +# Initial laser energy (at iteration 50) +if galilean: + filename_init = 'pml_x_galilean_plt000050' + energy_start = 4.439376199524034e-08 +else: + filename_init = 'pml_x_psatd_plt000050' + energy_start = 7.282940107273505e-08 # Check consistency of field energy diagnostics with initial energy above -ds = yt.load('pml_x_psatd_plt000050') +ds = yt.load(filename_init) all_data_level_0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() By = all_data_level_0['boxlib', 'By'].v.squeeze() @@ -43,10 +49,8 @@ print("relative error = " + str(error)) assert (error < tolerance) -########################## -### FINAL LASER ENERGY ### -########################## -ds = yt.load( filename ) +# Final laser energy +ds = yt.load(filename) all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() By = all_data_level_0['boxlib', 'By'].v.squeeze() @@ -58,8 +62,8 @@ energyB = np.sum(0.5 / scc.mu_0 * (Bx**2 + By**2 + Bz**2)) energy_end = energyE + energyB -reflectivity = energy_end / energy_start -reflectivity_max = 1e-06 +reflectivity = energy_end / energy_start_diags +reflectivity_max = 1e-6 print("reflectivity = " + str(reflectivity)) print("reflectivity_max = " + str(reflectivity_max)) @@ -70,7 +74,8 @@ sys.path.insert(0, '../../../../warpx/Examples/') from analysis_default_restart import check_restart -check_restart(filename) +if not galilean: + check_restart(filename) test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Python/pywarpx/LoadThirdParty.py b/Python/pywarpx/LoadThirdParty.py new file mode 100644 index 00000000000..ea62d558eeb --- /dev/null +++ b/Python/pywarpx/LoadThirdParty.py @@ -0,0 +1,35 @@ +# This file is part of WarpX. +# +# Authors: Axel Huebl +# License: BSD-3-Clause-LBNL + +from ._libwarpx import libwarpx + + +def load_cupy(): + """ + This is a helper for + https://docs.cupy.dev/en/stable/user_guide/basic.html + """ + amr = libwarpx.amr + status = None + + if amr.Config.have_gpu: + try: + import cupy as cp + xp = cp + # Note: found and will use cupy + except ImportError: + status = "Warning: GPU found but cupy not available! Trying managed memory in numpy..." + import numpy as np + xp = np + if amr.Config.gpu_backend == "SYCL": + status = "Warning: SYCL GPU backend not yet implemented for Python" + import numpy as np + xp = np + + else: + import numpy as np + xp = np + # Note: found and will use numpy + return xp, status diff --git a/Python/pywarpx/__init__.py b/Python/pywarpx/__init__.py index 8b57f60fb79..f89926399ca 100644 --- a/Python/pywarpx/__init__.py +++ b/Python/pywarpx/__init__.py @@ -34,6 +34,7 @@ from .HybridPICModel import hybridpicmodel from .Interpolation import interpolation from .Lasers import lasers +from .LoadThirdParty import load_cupy from .PSATD import psatd from .Particles import newspecies, particles from .WarpX import warpx diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index ab66a464628..fd647287a53 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -65,9 +65,14 @@ class _MultiFABWrapper(object): Parameters ---------- + mf: MultiFab + The Multifab that is wrapped. + mf_name: string The name of the MultiFab to be accessed, the tag specified when the - MultiFab is allocated + MultiFab is allocated. The Multifab will be accessed anew from WarpX + everytime it is called if this argument is given instead of directly + providing the Multifab. level: int The refinement level @@ -77,7 +82,8 @@ class _MultiFABWrapper(object): Note that when True, the first n-ghost negative indices will refer to the lower ghost cells. """ - def __init__(self, mf_name, level, include_ghosts=False): + def __init__(self, mf=None, mf_name=None, level=0, include_ghosts=False): + self._mf = mf self.mf_name = mf_name self.level = level self.include_ghosts = include_ghosts @@ -99,10 +105,13 @@ def __iter__(self): @property def mf(self): - # Always fetch this anew in case the C++ MultiFab is recreated - warpx = libwarpx.libwarpx_so.get_instance() - # All MultiFab names have the level suffix - return warpx.multifab(f'{self.mf_name}[level={self.level}]') + if self._mf is not None: + return self._mf + else: + # Always fetch this anew in case the C++ MultiFab is recreated + warpx = libwarpx.libwarpx_so.get_instance() + # All MultiFab names have the level suffix + return warpx.multifab(f'{self.mf_name}[level={self.level}]') @property def shape(self): @@ -544,143 +553,144 @@ def max_index(self, *args): def norm0(self, *args): return self.mf.norm0(*args) + def ExWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_aux[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_aux[x]', level=level, include_ghosts=include_ghosts) def EyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_aux[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_aux[y]', level=level, include_ghosts=include_ghosts) def EzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_aux[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_aux[z]', level=level, include_ghosts=include_ghosts) def BxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_aux[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_aux[x]', level=level, include_ghosts=include_ghosts) def ByWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_aux[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_aux[y]', level=level, include_ghosts=include_ghosts) def BzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_aux[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_aux[z]', level=level, include_ghosts=include_ghosts) def JxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[x]', level=level, include_ghosts=include_ghosts) def JyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[y]', level=level, include_ghosts=include_ghosts) def JzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[z]', level=level, include_ghosts=include_ghosts) def ExFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_fp[x]', level=level, include_ghosts=include_ghosts) def EyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_fp[y]', level=level, include_ghosts=include_ghosts) def EzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_fp[z]', level=level, include_ghosts=include_ghosts) def BxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_fp[x]', level=level, include_ghosts=include_ghosts) def ByFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_fp[y]', level=level, include_ghosts=include_ghosts) def BzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_fp[z]', level=level, include_ghosts=include_ghosts) def JxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[x]', level=level, include_ghosts=include_ghosts) def JyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[y]', level=level, include_ghosts=include_ghosts) def JzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[z]', level=level, include_ghosts=include_ghosts) def RhoFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'rho_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='rho_fp', level=level, include_ghosts=include_ghosts) def PhiFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'phi_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='phi_fp', level=level, include_ghosts=include_ghosts) def FFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'F_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='F_fp', level=level, include_ghosts=include_ghosts) def GFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'G_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='G_fp', level=level, include_ghosts=include_ghosts) def AxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'vector_potential_fp_nodal[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[x]', level=level, include_ghosts=include_ghosts) def AyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'vector_potential_fp_nodal[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[y]', level=level, include_ghosts=include_ghosts) def AzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'vector_potential_fp_nodal[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[z]', level=level, include_ghosts=include_ghosts) def ExCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_cp[x]', level=level, include_ghosts=include_ghosts) def EyCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_cp[y]', level=level, include_ghosts=include_ghosts) def EzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_cp[z]', level=level, include_ghosts=include_ghosts) def BxCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_cp[x]', level=level, include_ghosts=include_ghosts) def ByCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_cp[y]', level=level, include_ghosts=include_ghosts) def BzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_cp[z]', level=level, include_ghosts=include_ghosts) def JxCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_cp[x]', level=level, include_ghosts=include_ghosts) def JyCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_cp[y]', level=level, include_ghosts=include_ghosts) def JzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_cp[z]', level=level, include_ghosts=include_ghosts) def RhoCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'rho_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='rho_cp', level=level, include_ghosts=include_ghosts) def FCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'F_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='F_cp', level=level, include_ghosts=include_ghosts) def GCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'G_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='G_cp', level=level, include_ghosts=include_ghosts) def EdgeLengthsxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_edge_lengths[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_edge_lengths[x]', level=level, include_ghosts=include_ghosts) def EdgeLengthsyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_edge_lengths[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_edge_lengths[y]', level=level, include_ghosts=include_ghosts) def EdgeLengthszWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_edge_lengths[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_edge_lengths[z]', level=level, include_ghosts=include_ghosts) def FaceAreasxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_face_areas[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_face_areas[x]', level=level, include_ghosts=include_ghosts) def FaceAreasyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_face_areas[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_face_areas[y]', level=level, include_ghosts=include_ghosts) def FaceAreaszWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_face_areas[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_face_areas[z]', level=level, include_ghosts=include_ghosts) def JxFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp_ampere[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp_ampere[x]', level=level, include_ghosts=include_ghosts) def JyFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp_ampere[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp_ampere[y]', level=level, include_ghosts=include_ghosts) def JzFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp_ampere[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp_ampere[z]', level=level, include_ghosts=include_ghosts) def ExFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper(mf_name='pml_E_fp[x]', level=level, include_ghosts=include_ghosts) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index aed05121668..c74549bcdb7 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -2,12 +2,13 @@ # # This file is part of WarpX. # -# Authors: David Grote, Roelof Groenewald +# Authors: David Grote, Roelof Groenewald, Axel Huebl # # License: BSD-3-Clause-LBNL import numpy as np +from .LoadThirdParty import load_cupy from ._libwarpx import libwarpx @@ -183,40 +184,63 @@ def add_real_comp(self, pid_name, comm=True): ''' self.particle_container.add_real_comp(pid_name, comm) - def get_particle_structs(self, level): + def get_particle_structs(self, level, copy_to_host=False): ''' - This returns a list of numpy arrays containing the particle struct data + This returns a list of numpy or cupy arrays containing the particle struct data on each tile for this process. The particle data is represented as a structured - numpy array and contains the particle 'x', 'y', 'z', and 'idcpu'. + array and contains the particle 'x', 'y', 'z', and 'idcpu'. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + Unless copy_to_host is specified, the data for the structs are + not copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. + + Note that cupy does not support structs: + https://github.com/cupy/cupy/issues/2031 + and will return arrays of binary blobs for the AoS (DP: "|V24"). If copied + to host via copy_to_host, we correct for the right numpy AoS type. Parameters ---------- level : int - The refinement level to reference + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. Returns ------- - List of numpy arrays + List of arrays The requested particle struct data ''' particle_data = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): - aos_arr = np.array(pti.aos(), copy=False) - particle_data.append(aos_arr) + if copy_to_host: + particle_data.append(pti.aos().to_numpy(copy=True)) + else: + if libwarpx.amr.Config.have_gpu: + libwarpx.amr.Print( + "get_particle_structs: cupy does not yet support structs. " + "https://github.com/cupy/cupy/issues/2031" + "Did you mean copy_to_host=True?" + ) + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy + particle_data.append(aos_arr) return particle_data - def get_particle_arrays(self, comp_name, level): + def get_particle_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy arrays containing the particle array data + This returns a list of numpy or cupy arrays containing the particle array data on each tile for this process. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + Unless copy_to_host is specified, the data for the arrays are not + copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. Parameters ---------- @@ -224,13 +248,17 @@ def get_particle_arrays(self, comp_name, level): comp_name : str The component of the array data that will be returned - level : int - The refinement level to reference + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. Returns ------- - List of numpy arrays + List of arrays The requested particle array data ''' comp_idx = self.particle_container.get_comp_index(comp_name) @@ -238,101 +266,226 @@ def get_particle_arrays(self, comp_name, level): data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - data_array.append(np.array(soa.GetRealData(comp_idx), copy=False)) + idx = soa.GetRealData(comp_idx) + if copy_to_host: + data_array.append(idx.to_numpy(copy=True)) + else: + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + + data_array.append(xp.array(idx, copy=False)) + return data_array - def get_particle_id(self, level=0): + def get_particle_id(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle 'id' + Return a list of numpy or cupy arrays containing the particle 'id' numbers on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle ids ''' - structs = self.get_particle_structs(level) + structs = self.get_particle_structs(level, copy_to_host) return [libwarpx.amr.unpack_ids(struct['cpuid']) for struct in structs] - def get_particle_cpu(self, level=0): + def get_particle_cpu(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle 'cpu' + Return a list of numpy or cupy arrays containing the particle 'cpu' numbers on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle cpus ''' - structs = self.get_particle_structs(level) + structs = self.get_particle_structs(level, copy_to_host) return [libwarpx.amr.unpack_cpus(struct['cpuid']) for struct in structs] - def get_particle_x(self, level=0): + def get_particle_x(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle 'x' + Return a list of numpy or cupy arrays containing the particle 'x' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle x position ''' - structs = self.get_particle_structs(level) + if copy_to_host: + # Use the numpy version of cosine + xp = np + else: + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == '3d' or libwarpx.geometry_dim == '2d': return [struct['x'] for struct in structs] elif libwarpx.geometry_dim == 'rz': - return [struct['x']*np.cos(theta) for struct, theta in zip(structs, self.get_particle_theta())] + theta = self.get_particle_theta(level, copy_to_host) + return [struct['x']*xp.cos(theta) for struct, theta in zip(structs, theta)] elif libwarpx.geometry_dim == '1d': raise Exception('get_particle_x: There is no x coordinate with 1D Cartesian') xp = property(get_particle_x) - def get_particle_y(self, level=0): + def get_particle_y(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle 'y' + Return a list of numpy or cupy arrays containing the particle 'y' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle y position ''' - structs = self.get_particle_structs(level) + if copy_to_host: + # Use the numpy version of sine + xp = np + else: + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == '3d': return [struct['y'] for struct in structs] elif libwarpx.geometry_dim == 'rz': - return [struct['x']*np.sin(theta) for struct, theta in zip(structs, self.get_particle_theta())] + theta = self.get_particle_theta(level, copy_to_host) + return [struct['x']*xp.sin(theta) for struct, theta in zip(structs, theta)] elif libwarpx.geometry_dim == '1d' or libwarpx.geometry_dim == '2d': raise Exception('get_particle_y: There is no y coordinate with 1D or 2D Cartesian') yp = property(get_particle_y) - def get_particle_r(self, level=0): + def get_particle_r(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle 'r' + Return a list of numpy or cupy arrays containing the particle 'r' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle r position ''' - structs = self.get_particle_structs(level) + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == 'rz': return [struct['x'] for struct in structs] elif libwarpx.geometry_dim == '3d': - return [np.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] + return [xp.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_r: There is no r coordinate with 1D or 2D Cartesian') rp = property(get_particle_r) - def get_particle_theta(self, level=0): + def get_particle_theta(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle + Return a list of numpy or cupy arrays containing the particle theta on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle theta position ''' + xp, cupy_status = load_cupy() + if libwarpx.geometry_dim == 'rz': - return self.get_particle_arrays('theta', level) + return self.get_particle_arrays('theta', level, copy_to_host) elif libwarpx.geometry_dim == '3d': - structs = self.get_particle_structs(level) - return [np.arctan2(struct['y'], struct['x']) for struct in structs] + structs = self.get_particle_structs(level, copy_to_host) + return [xp.arctan2(struct['y'], struct['x']) for struct in structs] elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian') thetap = property(get_particle_theta) - def get_particle_z(self, level=0): + def get_particle_z(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle 'z' + Return a list of numpy or cupy arrays containing the particle 'z' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle z position ''' - structs = self.get_particle_structs(level) + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == '3d': return [struct['z'] for struct in structs] elif libwarpx.geometry_dim == 'rz' or libwarpx.geometry_dim == '2d': @@ -341,45 +494,101 @@ def get_particle_z(self, level=0): return [struct['x'] for struct in structs] zp = property(get_particle_z) - def get_particle_weight(self, level=0): + def get_particle_weight(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle + Return a list of numpy or cupy arrays containing the particle weight on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle weight ''' - return self.get_particle_arrays('w', level) + return self.get_particle_arrays('w', level, copy_to_host=copy_to_host) wp = property(get_particle_weight) - def get_particle_ux(self, level=0): + def get_particle_ux(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle + Return a list of numpy or cupy arrays containing the particle x momentum on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle x momentum ''' - return self.get_particle_arrays('ux', level) + return self.get_particle_arrays('ux', level, copy_to_host=copy_to_host) uxp = property(get_particle_ux) - def get_particle_uy(self, level=0): + def get_particle_uy(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle + Return a list of numpy or cupy arrays containing the particle y momentum on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle y momentum ''' - return self.get_particle_arrays('uy', level) + return self.get_particle_arrays('uy', level, copy_to_host=copy_to_host) uyp = property(get_particle_uy) - def get_particle_uz(self, level=0): + def get_particle_uz(self, level=0, copy_to_host=False): ''' - - Return a list of numpy arrays containing the particle + Return a list of numpy or cupy arrays containing the particle z momentum on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle z momentum ''' - return self.get_particle_arrays('uz', level) + return self.get_particle_arrays('uz', level, copy_to_host=copy_to_host) uzp = property(get_particle_uz) def get_species_charge_sum(self, local=False): @@ -457,13 +666,13 @@ def get_particle_boundary_buffer_size(self, species_name, boundary, local=False) def get_particle_boundary_buffer_structs(self, species_name, boundary, level): ''' - This returns a list of numpy arrays containing the particle struct data + This returns a list of numpy or cupy arrays containing the particle struct data for a species that has been scraped by a specific simulation boundary. The - particle data is represented as a structured numpy array and contains the + particle data is represented as a structured array and contains the particle 'x', 'y', 'z', and 'idcpu'. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + The data for the arrays are not copied, but share the underlying + memory buffer with WarpX. The arrays are fully writeable. Parameters ---------- @@ -500,11 +709,11 @@ def get_particle_boundary_buffer_structs(self, species_name, boundary, level): def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level): ''' - This returns a list of numpy arrays containing the particle array data + This returns a list of numpy or cupy arrays containing the particle array data for a species that has been scraped by a specific simulation boundary. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + The data for the arrays are not copied, but share the underlying + memory buffer with WarpX. The arrays are fully writeable. Parameters ---------- @@ -524,6 +733,8 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) level : int Which AMR level to retrieve scraped particle data from. ''' + xp, cupy_status = load_cupy() + part_container = self.particle_buffer.get_particle_container( species_name, self._get_boundary_number(boundary) ) @@ -533,14 +744,14 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) comp_idx = part_container.num_int_comps() - 1 for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(np.array(soa.GetIntData(comp_idx), copy=False)) + data_array.append(xp.array(soa.GetIntData(comp_idx), copy=False)) else: mypc = libwarpx.warpx.multi_particle_container() sim_part_container_wrapper = mypc.get_particle_container_from_name(species_name) comp_idx = sim_part_container_wrapper.get_comp_index(comp_name) for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(np.array(soa.GetRealData(comp_idx), copy=False)) + data_array.append(xp.array(soa.GetRealData(comp_idx), copy=False)) return data_array diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index dbdebf452d1..9e88da7c66e 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -8,6 +8,7 @@ """Classes following the PICMI standard """ +from dataclasses import dataclass import os import re @@ -1680,6 +1681,9 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_amrex_the_arena_init_size: long int, optional The amount of memory in bytes to allocate in the Arena. + warpx_amrex_use_gpu_aware_mpi: bool, optional + Whether to use GPU-aware MPI communications + warpx_zmax_plasma_to_compute_max_step: float, optional Sets the simulation run time based on the maximum z value @@ -1704,6 +1708,28 @@ class Simulation(picmistandard.PICMI_Simulation): The domain will be chopped into the exact number of pieces in each dimension as specified by this parameter. https://warpx.readthedocs.io/en/latest/usage/parameters.html#distribution-across-mpi-ranks-and-parallelization https://warpx.readthedocs.io/en/latest/usage/domain_decomposition.html#simple-method + + warpx_sort_intervals: string, optional (defaults: -1 on CPU; 4 on GPU) + Using the Intervals parser syntax, this string defines the timesteps at which particles are sorted. If <=0, do not sort particles. + It is turned on on GPUs for performance reasons (to improve memory locality). + + warpx_sort_particles_for_deposition: bool, optional (default: true for the CUDA backend, otherwise false) + This option controls the type of sorting used if particle sorting is turned on, i.e. if sort_intervals is not <=0. + If `true`, particles will be sorted by cell to optimize deposition with many particles per cell, in the order `x` -> `y` -> `z` -> `ppc`. + If `false`, particles will be sorted by bin, using the sort_bin_size parameter below, in the order `ppc` -> `x` -> `y` -> `z`. + `true` is recommended for best performance on NVIDIA GPUs, especially if there are many particles per cell. + + warpx_sort_idx_type: list of int, optional (default: 0 0 0) + This controls the type of grid used to sort the particles when sort_particles_for_deposition is true. + Possible values are: + idx_type = {0, 0, 0}: Sort particles to a cell centered grid, + idx_type = {1, 1, 1}: Sort particles to a node centered grid, + idx_type = {2, 2, 2}: Compromise between a cell and node centered grid. + In 2D (XZ and RZ), only the first two elements are read. In 1D, only the first element is read. + + warpx_sort_bin_size: list of int, optional (default 1 1 1) + If `sort_intervals` is activated and `sort_particles_for_deposition` is false, particles are sorted in bins of `sort_bin_size` cells. + In 2D, only the first two elements are read. """ # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to @@ -1739,8 +1765,13 @@ def init(self, kw): self.amr_restart = kw.pop('warpx_amr_restart', None) self.amrex_the_arena_is_managed = kw.pop('warpx_amrex_the_arena_is_managed', None) self.amrex_the_arena_init_size = kw.pop('warpx_amrex_the_arena_init_size', None) + self.amrex_use_gpu_aware_mpi = kw.pop('warpx_amrex_use_gpu_aware_mpi', None) self.zmax_plasma_to_compute_max_step = kw.pop('warpx_zmax_plasma_to_compute_max_step', None) self.compute_max_step_from_btd = kw.pop('warpx_compute_max_step_from_btd', None) + self.sort_intervals = kw.pop('warpx_sort_intervals', None) + self.sort_particles_for_deposition = kw.pop('warpx_sort_particles_for_deposition', None) + self.sort_idx_type = kw.pop('warpx_sort_idx_type', None) + self.sort_bin_size = kw.pop('warpx_sort_bin_size', None) self.collisions = kw.pop('warpx_collisions', None) self.embedded_boundary = kw.pop('warpx_embedded_boundary', None) @@ -1769,6 +1800,11 @@ def initialize_inputs(self): pywarpx.warpx.zmax_plasma_to_compute_max_step = self.zmax_plasma_to_compute_max_step pywarpx.warpx.compute_max_step_from_btd = self.compute_max_step_from_btd + pywarpx.warpx.sort_intervals = self.sort_intervals + pywarpx.warpx.sort_particles_for_deposition = self.sort_particles_for_deposition + pywarpx.warpx.sort_idx_type = self.sort_idx_type + pywarpx.warpx.sort_bin_size = self.sort_bin_size + pywarpx.algo.current_deposition = self.current_deposition_algo pywarpx.algo.charge_deposition = self.charge_deposition_algo pywarpx.algo.field_gathering = self.field_gathering_algo @@ -1869,6 +1905,9 @@ def initialize_inputs(self): if self.amrex_the_arena_init_size is not None: pywarpx.amrex.the_arena_init_size = self.amrex_the_arena_init_size + if self.amrex_use_gpu_aware_mpi is not None: + pywarpx.amrex.use_gpu_aware_mpi = self.amrex_use_gpu_aware_mpi + def initialize_warpx(self, mpi_comm=None): if self.warpx_initialized: return @@ -1934,6 +1973,35 @@ def set_write_dir(self): self.diagnostic.file_prefix = os.path.join(write_dir, file_prefix) +@dataclass(frozen=True) +class ParticleFieldDiagnostic: + """ + Class holding particle field diagnostic information to be processed in FieldDiagnostic below. + + Parameters + ---------- + name: str + Name of particle field diagnostic. If a component of a vector field, for the openPMD viewer + to treat it as a vector, the coordinate (i.e x, y, z) should be the last character. + + func: parser str + Parser function to be calculated for each particle per cell. Should be of the form + f(x,y,z,ux,uy,uz) + + do_average: (0 or 1) optional, default 1 + Whether the diagnostic is averaged by the sum of particle weights in each cell + + filter: parser str, optional + Parser function returning a boolean for whether to include a particle in the diagnostic. + If not specified, all particles will be included. The function arguments are the same + as the `func` above. + """ + name: str + func: str + do_average: int = 1 + filter: str = None + + class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): """ See `Input Parameters `_ for more information. @@ -1964,13 +2032,14 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_dump_rz_modes: bool, optional Flag whether to dump the data for all RZ modes - warpx_lower_bound: vector of floats, optional - Lower corner of output fields - that is passed to .lower_bound + warpx_particle_fields_to_plot: list of ParticleFieldDiagnostics + List of ParticleFieldDiagnostic classes to install in the simulation. Error + checking is handled in the class itself. - warpx_upper_bound: vector of floats, optional - Upper corner of output fields - that is passed to .upper_bound + warpx_particle_fields_species: list of strings, optional + Species for which to calculate particle_fields_to_plot functions. Fields will + be calculated separately for each specified species. If not passed, default is + all of the available particle species. """ def init(self, kw): @@ -1984,8 +2053,8 @@ def init(self, kw): self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.dump_rz_modes = kw.pop('warpx_dump_rz_modes', None) - self.lower_bound = kw.pop('warpx_lower_bound', None) - self.upper_bound = kw.pop('warpx_upper_bound', None) + self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) + self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) def initialize_inputs(self): @@ -2063,6 +2132,27 @@ def initialize_inputs(self): fields_to_plot.sort() self.diagnostic.fields_to_plot = fields_to_plot + particle_fields_to_plot_names = list() + for pfd in self.particle_fields_to_plot: + if pfd.name in particle_fields_to_plot_names: + raise Exception('A particle fields name can not be repeated.') + particle_fields_to_plot_names.append(pfd.name) + self.diagnostic.__setattr__( + f'particle_fields.{pfd.name}(x,y,z,ux,uy,uz)', pfd.func + ) + self.diagnostic.__setattr__( + f'particle_fields.{pfd.name}.do_average', pfd.do_average + ) + self.diagnostic.__setattr__( + f'particle_fields.{pfd.name}.filter(x,y,z,ux,uy,uz)', pfd.filter + ) + + # --- Convert to a sorted list so that the order + # --- is the same on all processors. + particle_fields_to_plot_names.sort() + self.diagnostic.particle_fields_to_plot = particle_fields_to_plot_names + self.diagnostic.particle_fields_species = self.particle_fields_species + self.diagnostic.plot_raw_fields = self.plot_raw_fields self.diagnostic.plot_raw_fields_guards = self.plot_raw_fields_guards self.diagnostic.plot_finepatch = self.plot_finepatch @@ -2249,6 +2339,10 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, warpx_file_prefix: string, optional Passed to .file_prefix + warpx_intervals: integer or string + Selects the snapshots to be made, instead of using "num_snapshots" which + makes all snapshots. "num_snapshots" is ignored. + warpx_file_min_digits: integer, optional Passed to .file_min_digits @@ -2268,6 +2362,7 @@ def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) self.file_prefix = kw.pop('warpx_file_prefix', None) + self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.buffer_size = kw.pop('warpx_buffer_size', None) self.lower_bound = kw.pop('warpx_lower_bound', None) @@ -2286,10 +2381,15 @@ def initialize_inputs(self): self.diagnostic.diag_hi = self.upper_bound self.diagnostic.do_back_transformed_fields = 1 - self.diagnostic.num_snapshots_lab = self.num_snapshots self.diagnostic.dt_snapshots_lab = self.dt_snapshots self.diagnostic.buffer_size = self.buffer_size + # intervals and num_snapshots_lab cannot both be set + if self.intervals is not None: + self.diagnostic.intervals = self.intervals + else: + self.diagnostic.num_snapshots_lab = self.num_snapshots + self.diagnostic.do_back_transformed_particles = self.write_species # --- Use a set to ensure that fields don't get repeated. @@ -2352,6 +2452,10 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, warpx_file_prefix: string, optional Passed to .file_prefix + warpx_intervals: integer or string + Selects the snapshots to be made, instead of using "num_snapshots" which + makes all snapshots. "num_snapshots" is ignored. + warpx_file_min_digits: integer, optional Passed to .file_min_digits @@ -2365,6 +2469,7 @@ def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) self.file_prefix = kw.pop('warpx_file_prefix', None) + self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.buffer_size = kw.pop('warpx_buffer_size', None) self.write_fields = kw.pop('warpx_write_fields', None) @@ -2379,10 +2484,15 @@ def initialize_inputs(self): self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.do_back_transformed_particles = 1 - self.diagnostic.num_snapshots_lab = self.num_snapshots self.diagnostic.dt_snapshots_lab = self.dt_snapshots self.diagnostic.buffer_size = self.buffer_size + # intervals and num_snapshots_lab cannot both be set + if self.intervals is not None: + self.diagnostic.intervals = self.intervals + else: + self.diagnostic.num_snapshots_lab = self.num_snapshots + self.diagnostic.do_back_transformed_fields = self.write_fields self.set_write_dir() diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json new file mode 100644 index 00000000000..454afc73bcc --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json @@ -0,0 +1,7 @@ +{ + "lev=0": { + "Ez": 123800860242.32724, + "jz": 48549778373280.2, + "rho": 344278.3437340355 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json new file mode 100644 index 00000000000..8afc645639d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json @@ -0,0 +1,9 @@ +{ + "lev=0": { + "Ex": 3790984950257.2373, + "Ez": 3790984950257.2373, + "jx": 1.0093528745824196e+16, + "jz": 1.0093528745824196e+16, + "rho": 21098347.344494 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json new file mode 100644 index 00000000000..2613f971f91 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bt": 3.64571737083816, + "Er": 1300317001574.1577, + "Ez": 1815076954086.807, + "jr": 244937533603400.0, + "jz": 342537403264912.5, + "rho": 10096982.130957149 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json new file mode 100644 index 00000000000..1e5d41f6677 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Bx": 20.814308179643078, + "By": 20.814308179643234, + "Bz": 20.81430817964295, + "Ex": 82994788022201.23, + "Ey": 82994788022201.23, + "Ez": 82994788022201.23, + "jx": 6.343089944595606e+16, + "jy": 6.3430899445956056e+16, + "jz": 6.3430899445956056e+16, + "part_per_cell": 0.0, + "rho": 694435081.2275198 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json new file mode 100644 index 00000000000..2843e49ce22 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Bx": 14264658.38987597, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 4276056420659746.5, + "Ez": 762168740318568.1, + "jx": 0.0, + "jy": 7.47674123799233e+16, + "jz": 4.817762115932484e+17, + "rho": 1609691680.1267354 + } +} + diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json new file mode 100644 index 00000000000..85919660834 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "Bx": 7268241.144300916, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 2146970168314705.0, + "Ez": 370041408451418.1, + "jx": 0.0, + "jy": 3.4818196503065956e+16, + "jz": 2.3196405062669472e+17, + "rho": 775945622.8993922 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/collider_diagnostics.json b/Regression/Checksum/benchmarks_json/collider_diagnostics.json new file mode 100644 index 00000000000..06553a7ec19 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/collider_diagnostics.json @@ -0,0 +1,36 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0, + "rho": 0.0, + "rho_beam_e": 9.613059803999999e-19, + "rho_beam_p": 9.613059803999999e-19 + }, + "beam_e": { + "particle_momentum_x": 1.638554718442694e-22, + "particle_momentum_y": 2.184739624590259e-22, + "particle_momentum_z": 2.8401615119673365e-21, + "particle_opticalDepthQSR": 9.767741201839083e+00, + "particle_position_x": 8.5, + "particle_position_y": 8.5, + "particle_position_z": 7.5, + "particle_weight": 6.0 + }, + "beam_p": { + "particle_momentum_x": 1.638554718442694e-22, + "particle_momentum_y": 2.184739624590259e-22, + "particle_momentum_z": 2.8401615119673365e-21, + "particle_opticalDepthQSR": 8.773870620305825e+00, + "particle_position_x": 8.5, + "particle_position_y": 8.5, + "particle_position_z": 7.5, + "particle_weight": 6.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/pml_x_galilean.json b/Regression/Checksum/benchmarks_json/pml_x_galilean.json new file mode 100644 index 00000000000..b32eb7a6518 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/pml_x_galilean.json @@ -0,0 +1,12 @@ +{ + "lev=0": { + "Bx": 9.111505955244123e-09, + "By": 1.743610723263872e-08, + "Bz": 8.692076437162089e-09, + "Ex": 4.8377358061392215, + "Ey": 3.8096703194246215, + "Ez": 4.747325170136437, + "divE": 480170.26408400893, + "rho": 1.7297829145620754e-06 + } +} \ No newline at end of file diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index c40dac874df..c6caa6b915c 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 23.09 +branch = 2e99628138df3b5b0ecf50b0c1201d5547f821a0 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 65a1e65e723..e4c502d858c 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 23.09 +branch = 2e99628138df3b5b0ecf50b0c1201d5547f821a0 [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -209,6 +209,22 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py +[collider_diagnostics] +buildDir = . +inputFile = Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles +runtime_params = warpx.abort_on_warning_threshold=high +dim = 3 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=3 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py + [collisionISO] buildDir = . inputFile = Examples/Tests/collision/inputs_3d_isotropization @@ -1080,6 +1096,78 @@ particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png +[Langmuir_fluid_1D] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_1d +runtime_params = warpx.do_dynamic_scheduling=0 +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py +analysisOutputImage = langmuir_fluid_multi_1d_analysis.png + +[Langmuir_fluid_RZ] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_rz +runtime_params = warpx.do_dynamic_scheduling=0 +dim = 2 +addToCompileString = USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py +analysisOutputImage = langmuir_fluid_rz_analysis.png + +[Langmuir_fluid_2D] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_2d +runtime_params = warpx.do_dynamic_scheduling=0 +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 -DCMAKE_BUILD_TYPE=Debug +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py +analysisOutputImage = langmuir_fluid_multi_2d_analysis.png + +[Langmuir_fluid_multi] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_3d +runtime_params = warpx.do_dynamic_scheduling=0 +dim = 3 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=3 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py +analysisOutputImage = langmuir_fluid_multi_analysis.png + [Langmuir_multi_1d] buildDir = . inputFile = Examples/Tests/langmuir/inputs_1d @@ -1725,6 +1813,42 @@ compareParticles = 1 particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py +[LaserAcceleration_1d_fluid] +buildDir = . +inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d_fluids +runtime_params = +dim = 1 +addToCompileString = USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons ions +analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py + +[LaserAcceleration_1d_fluid_boosted] +buildDir = . +inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted +runtime_params = +dim = 1 +addToCompileString = USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons ions +analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py + [LaserAccelerationBoost] buildDir = . inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d_boost @@ -2738,6 +2862,22 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py +[pml_x_galilean] +buildDir = . +inputFile = Examples/Tests/pml/inputs_2d +runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot=Ex Ey Ez Bx By Bz rho divE warpx.cfl=0.7071067811865475 warpx.do_pml_dive_cleaning=1 warpx.do_pml_divb_cleaning=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium psatd.v_galilean=0. 0. 0.99 warpx.grid_type=collocated +dim = 2 +addToCompileString = USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py + [pml_x_psatd] buildDir = . inputFile = Examples/Tests/pml/inputs_2d diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index d34a2f73222..5fc3e2dd3dc 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -738,11 +738,11 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const RealVect dx{AMREX_D_DECL(geom->CellSize(0), geom->CellSize(1), geom->CellSize(2))}; // Get the cell-centered box, with guard cells BoxArray realspace_ba = ba; // Copy box - amrex::Vector const v_galilean_zero = {0., 0., 0.}; + amrex::Vector const v_galilean = WarpX::GetInstance().m_v_galilean; amrex::Vector const v_comoving_zero = {0., 0., 0.}; realspace_ba.enclosedCells().grow(nge); // cell-centered + guard cells spectral_solver_fp = std::make_unique(lev, realspace_ba, dm, - nox_fft, noy_fft, noz_fft, grid_type, v_galilean_zero, + nox_fft, noy_fft, noz_fft, grid_type, v_galilean, v_comoving_zero, dx, dt, in_pml, periodic_single_box, update_with_rho, fft_do_time_averaging, psatd_solution_type, J_in_time, rho_in_time, m_dive_cleaning, m_divb_cleaning); #endif @@ -846,11 +846,11 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const RealVect cdx{AMREX_D_DECL(cgeom->CellSize(0), cgeom->CellSize(1), cgeom->CellSize(2))}; // Get the cell-centered box, with guard cells BoxArray realspace_cba = cba; // Copy box - amrex::Vector const v_galilean_zero = {0., 0., 0.}; + amrex::Vector const v_galilean = WarpX::GetInstance().m_v_galilean; amrex::Vector const v_comoving_zero = {0., 0., 0.}; realspace_cba.enclosedCells().grow(nge); // cell-centered + guard cells spectral_solver_cp = std::make_unique(lev, realspace_cba, cdm, - nox_fft, noy_fft, noz_fft, grid_type, v_galilean_zero, + nox_fft, noy_fft, noz_fft, grid_type, v_galilean, v_comoving_zero, cdx, dt, in_pml, periodic_single_box, update_with_rho, fft_do_time_averaging, psatd_solution_type, J_in_time, rho_in_time, m_dive_cleaning, m_divb_cleaning); #endif diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index b79132e83db..34bd720be96 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1450,8 +1450,9 @@ BTDiagnostics::PrepareParticleDataForOutput() { // Check if the zslice is in domain const bool ZSliceInDomain = GetZSliceInDomainFlag (i_buffer, lev); - if (ZSliceInDomain) { - if ( m_totalParticles_in_buffer[i_buffer][i] == 0) { + const bool kindexInSnapshotBox = GetKIndexInSnapshotBoxFlag (i_buffer, lev); + if (kindexInSnapshotBox) { + if ( buffer_empty(i_buffer) ) { if (!m_do_back_transformed_fields || m_varnames_fields.empty()) { if ( m_buffer_flush_counter[i_buffer] == 0) { DefineSnapshotGeometry(i_buffer, lev); diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index be1e408e16f..2c51c67662d 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -83,7 +83,7 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::ParticleReal x = p.pos(0); const amrex::Real lx = (x - plo[0]) * dxi[0]; ii = static_cast(amrex::Math::floor(lx)); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) const amrex::ParticleReal y = p.pos(1); const amrex::Real ly = (y - plo[1]) * dxi[1]; jj = static_cast(amrex::Math::floor(ly)); diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 73feef4ed00..c36e31c0f93 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -7,6 +7,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #endif #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Particles/WarpXParticleContainer.H" #include "WarpX.H" @@ -41,6 +43,10 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i if (m_species_index == -1) { auto& mypc = warpx.GetPartContainer(); rho = mypc.GetChargeDensity(m_lev, true); + if (warpx.DoFluidSpecies()) { + auto& myfl = warpx.GetFluidContainer(); + myfl.DepositCharge(m_lev, *rho); + } } // Dump rho per species else { diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 766745f1714..2b95df5d876 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -115,11 +115,15 @@ Diagnostics::BaseReadParameters () if (!pfield_varnames_specified){ m_pfield_varnames = {}; } + #ifdef WARPX_DIM_RZ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - m_pfield_varnames.empty(), - "Input error: cannot use particle_fields_to_plot with RZ" - ); + if (pfield_varnames_specified){ + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + "Particle field diagnostics will output the 0th mode only.", + ablastr::warn_manager::WarnPriority::low + ); + } #endif // Get parser strings for particle fields and generate map of parsers @@ -127,6 +131,7 @@ Diagnostics::BaseReadParameters () std::string filter_parser_str; const amrex::ParmParse pp_diag_pfield(m_diag_name + ".particle_fields"); for (const auto& var : m_pfield_varnames) { + bool do_average = true; pp_diag_pfield.query((var + ".do_average").c_str(), do_average); m_pfield_do_average.push_back(do_average); diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 7abe923b316..d5109277de2 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -187,6 +187,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const int ncomp = ncomp_multimodefab; // This function is called multiple times, for different values of `lev` // but the `varnames` need only be updated once. + const bool update_varnames = (lev==0); if (update_varnames) { m_varnames.clear(); @@ -194,9 +195,14 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) m_varnames.reserve(n_rz); } - // Reser field functors + // Add functors for average particle data for each species + const auto nvar = static_cast(m_varnames_fields.size()); + const auto nspec = static_cast(m_pfield_species.size()); + const auto ntot = static_cast(nvar + m_pfield_varnames.size() * nspec); + + // Reset field functors m_all_field_functors[lev].clear(); - m_all_field_functors[lev].resize(m_varnames_fields.size()); + m_all_field_functors[lev].resize(ntot); // Boolean flag for whether the current density should be deposited before // diagnostic output @@ -322,6 +328,20 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) "Error: " + m_varnames_fields[comp] + " is not a known field output type in RZ geometry"); } } + + // Generate field functors for every particle field diagnostic for every species in m_pfield_species. + // The names of the diagnostics are output in the `[varname]_[species]` format. + for (int pcomp=0; pcomp(nullptr, + lev, m_crse_ratio, m_pfield_strings[pcomp], m_pfield_species_index[ispec], m_pfield_do_average[pcomp], + m_pfield_dofilter[pcomp], m_pfield_filter_strings[pcomp]); + if (update_varnames) { + AddRZModesToOutputNames(std::string(m_pfield_varnames[pcomp]) + "_" + std::string(m_pfield_species[ispec]), ncomp); + } + } + } + // Sum the number of components in input vector m_all_field_functors // and check that it corresponds to the number of components in m_varnames // and m_mf_output @@ -329,6 +349,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) for (int jj=0; jjnComp(); } + AMREX_ALWAYS_ASSERT( ncomp_from_src == m_varnames.size() ); #else amrex::ignore_unused(lev); diff --git a/Source/Diagnostics/ReducedDiags/CMakeLists.txt b/Source/Diagnostics/ReducedDiags/CMakeLists.txt index b2a31267087..e63764bc24f 100644 --- a/Source/Diagnostics/ReducedDiags/CMakeLists.txt +++ b/Source/Diagnostics/ReducedDiags/CMakeLists.txt @@ -3,6 +3,7 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BeamRelevant.cpp + ColliderRelevant.cpp FieldEnergy.cpp FieldProbe.cpp FieldProbeParticleContainer.cpp diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H new file mode 100644 index 00000000000..1998c56812d --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H @@ -0,0 +1,61 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Arianna Formenti + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DIAGNOSTICS_REDUCEDDIAGS_COLLIDERRELEVANT_H_ +#define WARPX_DIAGNOSTICS_REDUCEDDIAGS_COLLIDERRELEVANT_H_ + +#include "ReducedDiags.H" + +#include +#include +#include + +/** + * This class contains diagnostics that are relevant to colliders. + */ +class ColliderRelevant : public ReducedDiags +{ +public: + + /** + * constructor + * @param[in] rd_name reduced diags names + */ + ColliderRelevant(std::string rd_name); + + /// name of the two colliding species + std::vector m_beam_name; + + /** + * \brief This function computes collider-relevant diagnostics. + * @param[in] step current time step + * + * [0]step, [1]time, [2]dL/dt, + * for first species: + * [3]chi_min, [4]chi_ave, [5] chi_max, + * [6]x_ave, [7]x_std, + * [8]y_ave, [9]y_std, + * [10]thetax_min, [11]thetax_ave, [12]thetax_max, [13]thetax_std, + * [14]thetay_min, [15]thetay_ave, [16]thetay_max, [17]thetay_std + * same for second species follows. + */ + void ComputeDiags(int step) override final; + +private: + /// auxiliary structure to store headers and indices of the reduced diagnostics + struct aux_header_index + { + std::string header; + int idx; + }; + + /// map to store header texts and indices of the reduced diagnostics + std::map m_headers_indices; +}; + +#endif // WARPX_DIAGNOSTICS_REDUCEDDIAGS_COLLIDERRELEVANT_H_ diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp new file mode 100644 index 00000000000..1e2c16ac737 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -0,0 +1,548 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Arianna Formenti, Yinjian Zhao + * License: BSD-3-Clause-LBNL + */ +#include "ColliderRelevant.H" + +#include "Diagnostics/ReducedDiags/ReducedDiags.H" +#if (defined WARPX_QED) +# include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" +#endif +#include "Particles/Gather/FieldGather.H" +#include "Particles/Gather/GetExternalFields.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/SpeciesPhysicalProperties.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/WarpXConst.H" +#include "Utils/TextMsg.H" +#include "WarpX.H" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace amrex; + +ColliderRelevant::ColliderRelevant (std::string rd_name) +: ReducedDiags{std::move(rd_name)} +{ + // read colliding species names - must be 2 + amrex::ParmParse pp_rd_name(m_rd_name); + pp_rd_name.getarr("species", m_beam_name); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_beam_name.size() == 2u, + "Collider-relevant diagnostics must involve exactly two species"); + + // RZ coordinate is not supported +#if (defined WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE( + "Collider-relevant diagnostics do not work in RZ geometry."); +#endif + + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + "The collider-relevant reduced diagnostic is meant for \ + colliding species propagating along the z direction.", + ablastr::warn_manager::WarnPriority::low); + + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + "The collider-relevant reduced diagnostic only considers the \ + coarsest level of refinement for the calculations involving chi.", + ablastr::warn_manager::WarnPriority::low); + + // get WarpX class object + auto& warpx = WarpX::GetInstance(); + + // get MultiParticleContainer class object + const MultiParticleContainer& mypc = warpx.GetPartContainer(); + + // loop over species + for (int i_s = 0; i_s < 2; ++i_s) + { + // get WarpXParticleContainer class object + const WarpXParticleContainer& myspc = mypc.GetParticleContainerFromName(m_beam_name[i_s]); + + // get charge + amrex::ParticleReal const q = myspc.getCharge(); + + // photon number density is not available yet + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + q!=amrex::Real(0.0), + "Collider-relevant diagnostic does not work for neutral species yet"); + } + + // function to fill a vector with diags names and create corresponding entry in header + std::vector all_diag_names; + auto add_diag = [&,c=0]( + const std::string& name, const std::string& header) mutable { + m_headers_indices[name] = aux_header_index{header, c++}; + all_diag_names.push_back(name); + }; + +#if (defined WARPX_DIM_3D) + add_diag("dL_dt", "dL_dt(m^-2*s^-1)"); +#elif (defined WARPX_DIM_XZ) + add_diag("dL_dt", "dL_dt(m^-1*s^-1)"); +#else + add_diag("dL_dt", "dL_dt(s^-1)"); +#endif + + // loop over species + for (int i_s = 0; i_s < 2; ++i_s) + { + // get WarpXParticleContainer class object + const WarpXParticleContainer& myspc = mypc.GetParticleContainerFromName(m_beam_name[i_s]); + + if (myspc.DoQED()){ + add_diag("chimin_"+m_beam_name[i_s], "chi_min_"+m_beam_name[i_s]+"()"); + add_diag("chiave_"+m_beam_name[i_s], "chi_ave_"+m_beam_name[i_s]+"()"); + add_diag("chimax_"+m_beam_name[i_s], "chi_max_"+m_beam_name[i_s]+"()"); + } +#if (defined WARPX_DIM_3D) + add_diag("x_ave_"+m_beam_name[i_s], "x_ave_"+m_beam_name[i_s]+"(m)"); + add_diag("x_std_"+m_beam_name[i_s], "x_std_"+m_beam_name[i_s]+"(m)"); + add_diag("y_ave_"+m_beam_name[i_s], "y_ave_"+m_beam_name[i_s]+"(m)"); + add_diag("y_std_"+m_beam_name[i_s], "y_std_"+m_beam_name[i_s]+"(m)"); + add_diag("thetax_min_"+m_beam_name[i_s], "theta_x_min_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetax_ave_"+m_beam_name[i_s], "theta_x_ave_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetax_max_"+m_beam_name[i_s], "theta_x_max_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetax_std_"+m_beam_name[i_s], "theta_x_std_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetay_min_"+m_beam_name[i_s], "theta_y_min_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetay_ave_"+m_beam_name[i_s], "theta_y_ave_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetay_max_"+m_beam_name[i_s], "theta_y_max_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetay_std_"+m_beam_name[i_s], "theta_y_std_"+m_beam_name[i_s]+"(rad)"); + +#elif (defined WARPX_DIM_XZ) + add_diag("x_ave_"+m_beam_name[i_s], "x_ave_"+m_beam_name[i_s]+"(m)"); + add_diag("x_std_"+m_beam_name[i_s], "x_std_"+m_beam_name[i_s]+"(m)"); + add_diag("thetax_min_"+m_beam_name[i_s], "theta_x_min_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetax_ave_"+m_beam_name[i_s], "theta_x_ave_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetax_max_"+m_beam_name[i_s], "theta_x_max_"+m_beam_name[i_s]+"(rad)"); + add_diag("thetax_std_"+m_beam_name[i_s], "theta_x_std_"+m_beam_name[i_s]+"(rad)"); +#endif + m_data.resize(all_diag_names.size()); + } + + if (amrex::ParallelDescriptor::IOProcessor()) + { + if ( m_write_header ) + { + // open file + std::ofstream ofs; + ofs.open(m_path + m_rd_name + "." + m_extension, + std::ofstream::out | std::ofstream::app); + // write header row + int off = 0; + ofs << "#"; + ofs << "[" << off++ << "]step()"; + ofs << m_sep; + ofs << "[" << off++ << "]time(s)"; + for (const auto& name : all_diag_names){ + const auto& el = m_headers_indices[name]; + ofs << m_sep << "[" << el.idx + off << "]" << el.header; + } + ofs << std::endl; + // close file + ofs.close(); + } + } +} + +void ColliderRelevant::ComputeDiags (int step) +{ +#if defined(WARPX_DIM_RZ) + amrex::ignore_unused(step); +#else + + // Judge if the diags should be done + if (!m_intervals.contains(step+1)) { return; } + + // get MultiParticleContainer class object + const MultiParticleContainer& mypc = WarpX::GetInstance().GetPartContainer(); + + // get a reference to WarpX instance + auto& warpx = WarpX::GetInstance(); + + // get cell volume + amrex::Geometry const & geom = warpx.Geom(0); + amrex::Real dV = AMREX_D_TERM(geom.CellSize(0), *geom.CellSize(1), *geom.CellSize(2)); + + const auto get_idx = [&](const std::string& name){ + return m_headers_indices.at(name).idx; + }; + + std::array, 2> num_dens; + + // loop over species + for (int i_s = 0; i_s < 2; ++i_s) + { + // get WarpXParticleContainer class object + WarpXParticleContainer& myspc = mypc.GetParticleContainerFromName(m_beam_name[i_s]); + // get charge + amrex::ParticleReal const q = myspc.getCharge(); + + using PType = typename WarpXParticleContainer::SuperParticleType; + + num_dens[i_s] = myspc.GetChargeDensity(0); + num_dens[i_s]->mult(1./q); + +#if defined(WARPX_DIM_1D_Z) + // w_tot + amrex::Real w_tot = ReduceSum( myspc, + [=] AMREX_GPU_HOST_DEVICE (const PType& p) + { + return p.rdata(PIdx::w); + }); + amrex::ParallelDescriptor::ReduceRealSum(w_tot); +#elif defined(WARPX_DIM_XZ) + // w_tot + // x_ave, + // thetax_min, thetax_ave, thetax_max + amrex::ReduceOps reduce_ops; + auto r = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple + { + const amrex::Real w = p.rdata(PIdx::w); + const amrex::Real x = p.pos(0); + const amrex::Real ux = p.rdata(PIdx::ux); + const amrex::Real uz = p.rdata(PIdx::uz); + const amrex::Real thetax = std::atan2(ux, uz); + return {w, w*x, thetax, w*thetax, thetax}; + }, + reduce_ops); + + amrex::Real w_tot = amrex::get<0>(r); + amrex::Real x_ave = amrex::get<1>(r); + amrex::Real thetax_min = amrex::get<2>(r); + amrex::Real thetax_ave = amrex::get<3>(r); + amrex::Real thetax_max = amrex::get<4>(r); + + amrex::ParallelDescriptor::ReduceRealSum({w_tot, x_ave, thetax_ave}); + amrex::ParallelDescriptor::ReduceRealMin({thetax_min}); + amrex::ParallelDescriptor::ReduceRealMax({thetax_max}); + + // x_std, thetax_std + amrex::Real x_std = 0.0_rt; + amrex::Real thetax_std = 0.0_rt; + + if (w_tot > 0.0_rt) + { + x_ave = x_ave / w_tot; + thetax_ave = thetax_ave / w_tot; + + amrex::ReduceOps reduce_ops_std; + auto r_std = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple + { + const amrex::Real w = p.rdata(PIdx::w); + const amrex::Real x = p.pos(0); + const amrex::Real ux = p.rdata(PIdx::ux); + const amrex::Real uz = p.rdata(PIdx::uz); + const amrex::Real thetax = std::atan2(ux, uz); + const amrex::Real tmp1 = (x - x_ave)*(x - x_ave)*w; + const amrex::Real tmp2 = (thetax - thetax_ave)*(thetax - thetax_ave)*w; + return {tmp1, tmp2}; + }, + reduce_ops_std); + + x_std = amrex::get<0>(r_std); + thetax_std = amrex::get<1>(r_std); + + amrex::ParallelDescriptor::ReduceRealSum({x_std, thetax_std}); + + x_std = std::sqrt(x_std / w_tot); + thetax_std = std::sqrt(thetax_std / w_tot); + } + + m_data[get_idx("x_ave_"+m_beam_name[i_s])] = x_ave; + m_data[get_idx("x_std_"+m_beam_name[i_s])] = x_std; + m_data[get_idx("thetax_min_"+m_beam_name[i_s])] = thetax_min; + m_data[get_idx("thetax_ave_"+m_beam_name[i_s])] = thetax_ave; + m_data[get_idx("thetax_max_"+m_beam_name[i_s])] = thetax_max; + m_data[get_idx("thetax_std_"+m_beam_name[i_s])] = thetax_std; +#elif defined(WARPX_DIM_3D) + // w_tot + // x_ave, y_ave, + // thetax_min, thetax_ave, thetax_max + // thetay_min, thetay_ave, thetay_max + amrex::ReduceOps reduce_ops; + auto r = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple + { + const amrex::Real w = p.rdata(PIdx::w); + const amrex::Real x = p.pos(0); + const amrex::Real y = p.pos(1); + const amrex::Real ux = p.rdata(PIdx::ux); + const amrex::Real uy = p.rdata(PIdx::uy); + const amrex::Real uz = p.rdata(PIdx::uz); + const amrex::Real thetax = std::atan2(ux, uz); + const amrex::Real thetay = std::atan2(uy, uz); + return {w, w*x, w*y, + thetax, w*thetax, thetax, + thetay, w*thetay, thetay}; + }, + reduce_ops); + + amrex::Real w_tot = amrex::get<0>(r); + amrex::Real x_ave = amrex::get<1>(r); + amrex::Real y_ave = amrex::get<2>(r); + amrex::Real thetax_min = amrex::get<3>(r); + amrex::Real thetax_ave = amrex::get<4>(r); + amrex::Real thetax_max = amrex::get<5>(r); + amrex::Real thetay_min = amrex::get<6>(r); + amrex::Real thetay_ave = amrex::get<7>(r); + amrex::Real thetay_max = amrex::get<8>(r); + + amrex::ParallelDescriptor::ReduceRealSum({w_tot, x_ave, y_ave, thetax_ave, thetay_ave}); + amrex::ParallelDescriptor::ReduceRealMin({thetax_min, thetay_min}); + amrex::ParallelDescriptor::ReduceRealMax({thetax_max, thetay_max}); + + // x_std, y_std, thetax_std, thetay_std + amrex::Real x_std = 0.0_rt; + amrex::Real y_std = 0.0_rt; + amrex::Real thetax_std = 0.0_rt; + amrex::Real thetay_std = 0.0_rt; + + if (w_tot > 0.0_rt) + { + x_ave = x_ave / w_tot; + y_ave = y_ave / w_tot; + thetax_ave = thetax_ave / w_tot; + thetay_ave = thetay_ave / w_tot; + + amrex::ReduceOps reduce_ops_std; + auto r_std = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple + { + const amrex::Real w = p.rdata(PIdx::w); + const amrex::Real x = p.pos(0); + const amrex::Real ux = p.rdata(PIdx::ux); + const amrex::Real y = p.pos(1); + const amrex::Real uy = p.rdata(PIdx::uy); + const amrex::Real uz = p.rdata(PIdx::uz); + const amrex::Real thetax = std::atan2(ux, uz); + const amrex::Real thetay = std::atan2(uy, uz); + const amrex::Real tmp1 = (x - x_ave)*(x - x_ave)*w; + const amrex::Real tmp2 = (y - y_ave)*(y - y_ave)*w; + const amrex::Real tmp3 = (thetax - thetax_ave)*(thetax - thetax_ave)*w; + const amrex::Real tmp4 = (thetay - thetay_ave)*(thetay - thetay_ave)*w; + return {tmp1, tmp2, tmp3, tmp4}; + }, + reduce_ops_std); + + x_std = amrex::get<0>(r_std); + y_std = amrex::get<1>(r_std); + thetax_std = amrex::get<2>(r_std); + thetay_std = amrex::get<3>(r_std); + + amrex::ParallelDescriptor::ReduceRealSum({x_std, y_std, thetax_std, thetay_std}); + + x_std = std::sqrt(x_std / w_tot); + y_std = std::sqrt(y_std / w_tot); + thetax_std = std::sqrt(thetax_std / w_tot); + thetay_std = std::sqrt(thetay_std / w_tot); + } + + m_data[get_idx("x_ave_"+m_beam_name[i_s])] = x_ave; + m_data[get_idx("x_std_"+m_beam_name[i_s])] = x_std; + m_data[get_idx("y_ave_"+m_beam_name[i_s])] = y_ave; + m_data[get_idx("y_std_"+m_beam_name[i_s])] = y_std; + m_data[get_idx("thetax_min_"+m_beam_name[i_s])] = thetax_min; + m_data[get_idx("thetax_ave_"+m_beam_name[i_s])] = thetax_ave; + m_data[get_idx("thetax_max_"+m_beam_name[i_s])] = thetax_max; + m_data[get_idx("thetax_std_"+m_beam_name[i_s])] = thetax_std; + m_data[get_idx("thetay_min_"+m_beam_name[i_s])] = thetay_min; + m_data[get_idx("thetay_ave_"+m_beam_name[i_s])] = thetay_ave; + m_data[get_idx("thetay_max_"+m_beam_name[i_s])] = thetay_max; + m_data[get_idx("thetay_std_"+m_beam_name[i_s])] = thetay_std; +#endif + +#if (defined WARPX_QED) + // get mass + amrex::ParticleReal m = myspc.getMass(); + const bool is_photon = myspc.AmIA(); + if (is_photon) { + m = PhysConst::m_e; + } + + // compute chimin, chiave and chimax + amrex::Real chimin_f = 0.0_rt; + amrex::Real chimax_f = 0.0_rt; + amrex::Real chiave_f = 0.0_rt; + + if (myspc.DoQED()) + { + // define variables in preparation for field gatheeduce_data.value()ring + const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + const int nox = WarpX::nox; + const bool galerkin_interpolation = WarpX::galerkin_interpolation; + const amrex::IntVect ngEB = warpx.getngEB(); + + // TODO loop over refinement levels: for (int lev = 0; lev <= level_number; ++lev) + const int lev = 0; + + // define variables in preparation for field gathering + const std::array& dx = WarpX::CellSize(std::max(lev, 0)); + const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::MultiFab & Ex = warpx.getEfield(lev,0); + const amrex::MultiFab & Ey = warpx.getEfield(lev,1); + const amrex::MultiFab & Ez = warpx.getEfield(lev,2); + const amrex::MultiFab & Bx = warpx.getBfield(lev,0); + const amrex::MultiFab & By = warpx.getBfield(lev,1); + const amrex::MultiFab & Bz = warpx.getBfield(lev,2); + + // declare reduce_op + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + + // Loop over boxes + for (WarpXParIter pti(myspc, lev); pti.isValid(); ++pti) + { + const auto GetPosition = GetParticlePosition(pti); + // get particle arrays + amrex::ParticleReal* const AMREX_RESTRICT ux = pti.GetAttribs()[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = pti.GetAttribs()[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = pti.GetAttribs()[PIdx::uz].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT w = pti.GetAttribs()[PIdx::w].dataPtr(); + // declare external fields + const int offset = 0; + const auto getExternalEB = GetExternalEBField(pti, offset); + // define variables in preparation for field gathering + amrex::Box box = pti.tilebox(); + box.grow(ngEB); + const amrex::Dim3 lo = amrex::lbound(box); + const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::Array4 & ex_arr = Ex[pti].array(); + const amrex::Array4 & ey_arr = Ey[pti].array(); + const amrex::Array4 & ez_arr = Ez[pti].array(); + const amrex::Array4 & bx_arr = Bx[pti].array(); + const amrex::Array4 & by_arr = By[pti].array(); + const amrex::Array4 & bz_arr = Bz[pti].array(); + const amrex::IndexType ex_type = Ex[pti].box().ixType(); + const amrex::IndexType ey_type = Ey[pti].box().ixType(); + const amrex::IndexType ez_type = Ez[pti].box().ixType(); + const amrex::IndexType bx_type = Bx[pti].box().ixType(); + const amrex::IndexType by_type = By[pti].box().ixType(); + const amrex::IndexType bz_type = Bz[pti].box().ixType(); + + // evaluate reduce_op + reduce_op.eval(pti.numParticles(), reduce_data, + [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple + { + // get external fields + amrex::ParticleReal xp, yp, zp; + GetPosition(i, xp, yp, zp); + amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; + amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + getExternalEB(i, ex, ey, ez, bx, by, bz); + + // gather E and B + doGatherShapeN(xp, yp, zp, + ex, ey, ez, bx, by, bz, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, + bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, + n_rz_azimuthal_modes, nox, galerkin_interpolation); + // compute chi + amrex::Real chi = 0.0_rt; + if (is_photon) { + chi = QedUtils::chi_photon(ux[i]*m, uy[i]*m, uz[i]*m, + ex, ey, ez, bx, by, bz); + } else { + chi = QedUtils::chi_ele_pos(ux[i]*m, uy[i]*m, uz[i]*m, + ex, ey, ez, bx, by, bz); + } + return {chi, chi, chi*w[i]}; + }); + } + auto val = reduce_data.value(); + chimin_f = get<0>(val); + chimax_f = get<1>(val); + chiave_f = get<2>(val); + amrex::ParallelDescriptor::ReduceRealMin(chimin_f); + amrex::ParallelDescriptor::ReduceRealMax(chimax_f); + amrex::ParallelDescriptor::ReduceRealSum(chiave_f); + + m_data[get_idx("chimin_"+m_beam_name[i_s])] = chimin_f; + m_data[get_idx("chiave_"+m_beam_name[i_s])] = chiave_f/w_tot; + m_data[get_idx("chimax_"+m_beam_name[i_s])] = chimax_f; + } +#endif + } // end loop over species + + // make density MultiFabs from nodal to cell centered + amrex::BoxArray ba = warpx.boxArray(0); + amrex::DistributionMapping dmap = warpx.DistributionMap(0); + constexpr int ncomp = 1; + constexpr int ngrow = 0; + amrex::MultiFab mf_dst1(ba.convert(amrex::IntVect::TheCellVector()), dmap, ncomp, ngrow); + amrex::MultiFab mf_dst2(ba.convert(amrex::IntVect::TheCellVector()), dmap, ncomp, ngrow); + ablastr::coarsen::sample::Coarsen(mf_dst1, *num_dens[0], 0, 0, ncomp, ngrow); + ablastr::coarsen::sample::Coarsen(mf_dst2, *num_dens[1], 0, 0, ncomp, ngrow); + + // compute luminosity + amrex::Real const n1_dot_n2 = amrex::MultiFab::Dot(mf_dst1, 0, mf_dst2, 0, 1, 0); + amrex::Real const lumi = 2. * PhysConst::c * n1_dot_n2 * dV; + m_data[get_idx("dL_dt")] = lumi; +#endif // not RZ +} diff --git a/Source/Diagnostics/ReducedDiags/Make.package b/Source/Diagnostics/ReducedDiags/Make.package index f265e302d3a..a1f08a24da3 100644 --- a/Source/Diagnostics/ReducedDiags/Make.package +++ b/Source/Diagnostics/ReducedDiags/Make.package @@ -7,6 +7,7 @@ CEXE_sources += FieldProbe.cpp CEXE_sources += FieldProbeParticleContainer.cpp CEXE_sources += FieldMomentum.cpp CEXE_sources += BeamRelevant.cpp +CEXE_sources += ColliderRelevant.cpp CEXE_sources += LoadBalanceCosts.cpp CEXE_sources += LoadBalanceEfficiency.cpp CEXE_sources += ParticleHistogram.cpp diff --git a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp index d895c4caa23..73e63aeb4d3 100644 --- a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp @@ -8,6 +8,7 @@ #include "BeamRelevant.H" #include "ChargeOnEB.H" +#include "ColliderRelevant.H" #include "FieldEnergy.H" #include "FieldMaximum.H" #include "FieldProbe.H" @@ -59,6 +60,7 @@ MultiReducedDiags::MultiReducedDiags () {"FieldReduction", [](CS s){return std::make_unique(s);}}, {"RhoMaximum", [](CS s){return std::make_unique(s);}}, {"BeamRelevant", [](CS s){return std::make_unique(s);}}, + {"ColliderRelevant", [](CS s){return std::make_unique(s);}}, {"LoadBalanceCosts", [](CS s){return std::make_unique(s);}}, {"LoadBalanceEfficiency", [](CS s){return std::make_unique(s);}}, {"ParticleHistogram", [](CS s){return std::make_unique(s);}}, diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 77377c7d5ed..758b48b7262 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -24,9 +24,8 @@ using namespace amrex; // constructor ReducedDiags::ReducedDiags (std::string rd_name) + : m_rd_name(std::move(rd_name)) { - m_rd_name = rd_name; - BackwardCompatibility(); const ParmParse pp_rd_name(m_rd_name); diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 14b2a994a3f..40fc431f4cd 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -24,6 +24,8 @@ #endif #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Particles/ParticleBoundaryBuffer.H" #include "Python/WarpX_py.H" #include "Utils/TextMsg.H" @@ -1037,6 +1039,12 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, else ApplySubcyclingScalingToCurrentDensity(current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), n_subcycle_current, lev); } + if (do_fluid_species) { + myfl->Evolve(lev, + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], + rho_fp[lev].get(),*current_x, *current_y, *current_z, cur_time, skip_deposition); + } } } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a6932f223e2..216e80f7494 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -7,6 +7,8 @@ #include "WarpX.H" #include "FieldSolver/ElectrostaticSolver.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" @@ -209,6 +211,10 @@ WarpX::AddSpaceChargeFieldLabFrame () // Deposit particle charge density (source of Poisson solver) mypc->DepositCharge(rho_fp, 0.0_rt); + if (do_fluid_species) { + int const lev = 0; + myfl->DepositCharge( lev, *rho_fp[lev] ); + } SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels #ifndef WARPX_DIM_RZ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 7f50d1cd8ea..cef6331888e 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -21,27 +21,59 @@ #if WARPX_USE_PSATD -/* \brief Class that updates the field in spectral space +/* + * \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ class PsatdAlgorithmPml : public SpectralBaseAlgorithm { public: - PsatdAlgorithmPml(const SpectralKSpace& spectral_kspace, - const amrex::DistributionMapping& dm, - const SpectralFieldIndex& spectral_index, - int norder_x, int norder_y, - int norder_z, short grid_type, - amrex::Real dt, - bool dive_cleaning, - bool divb_cleaning); - void InitializeSpectralCoefficients( + /** + * \brief Constructor of the class PsatdAlgorithmPml + * + * \param[in] spectral_kspace Spectral space + * \param[in] dm Distribution mapping + * \param[in] spectral_index Object containing indices to access data in spectral space + * \param[in] norder_x Order of the spectral solver along x + * \param[in] norder_y Order of the spectral solver along y + * \param[in] norder_z Order of the spectral solver along z + * \param[in] grid_type Type of grid (collocated or not) + * \param[in] v_galilean Galilean velocity + * \param[in] dt Time step of the simulation + * \param[in] dive_cleaning Whether to use divergence correction for E (F term) + * \param[in] divb_cleaning Whether to use divergence correction for B (G term) + */ + PsatdAlgorithmPml( const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, - amrex::Real dt); + const SpectralFieldIndex& spectral_index, + int norder_x, + int norder_y, + int norder_z, + short grid_type, + const amrex::Vector& v_galilean, + amrex::Real dt, + bool dive_cleaning, + bool divb_cleaning); + + /** + * \brief Initializes the coefficients used in \c pushSpectralFields + * to update the E and B fields + * + * \param[in] spectral_kspace Spectral space + * \param[in] dm Distribution mapping + */ + void InitializeSpectralCoefficients( + const SpectralKSpace& spectral_kspace, + const amrex::DistributionMapping& dm); - // Redefine functions from base class + /** + * \brief Updates the E and B fields in spectral space, + * according to the relevant PSATD equations + * + * \param[in,out] f All fields in spectral space + */ virtual void pushSpectralFields(SpectralFieldData& f) const override final; /** @@ -67,10 +99,20 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm virtual void VayDeposition (SpectralFieldData& field_data) override final; private: + SpectralRealCoefficients C_coef, S_ck_coef, inv_k2_coef; + SpectralComplexCoefficients T2_coef; + // Centered modified finite-order k vectors + KVectorComponent modified_kx_vec_centered; +#if defined(WARPX_DIM_3D) + KVectorComponent modified_ky_vec_centered; +#endif + KVectorComponent modified_kz_vec_centered; + amrex::Vector m_v_galilean; amrex::Real m_dt; bool m_dive_cleaning; bool m_divb_cleaning; + bool m_is_galilean; }; #endif // WARPX_USE_PSATD diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index 1a4039915e0..be91d920cba 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -9,6 +9,7 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpX_Complex.H" @@ -29,59 +30,85 @@ using namespace amrex; -/* \brief Initialize coefficients for the update equation */ -PsatdAlgorithmPml::PsatdAlgorithmPml(const SpectralKSpace& spectral_kspace, - const DistributionMapping& dm, - const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, - const Real dt, - const bool dive_cleaning, const bool divb_cleaning) - // Initialize members of base class - : SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), - m_dt(dt), - m_dive_cleaning(dive_cleaning), - m_divb_cleaning(divb_cleaning) +PsatdAlgorithmPml::PsatdAlgorithmPml( + const SpectralKSpace& spectral_kspace, + const DistributionMapping& dm, + const SpectralFieldIndex& spectral_index, + int norder_x, + int norder_y, + int norder_z, + short grid_type, + const amrex::Vector& v_galilean, + Real dt, + bool dive_cleaning, + bool divb_cleaning) : + SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), + // Initialize the centered finite-order modified k vectors: + // these are computed always with the assumption of centered grids + // (argument grid_type=GridType::Collocated), for both collocated and staggered grids + modified_kx_vec_centered(spectral_kspace.getModifiedKComponent(dm, 0, norder_x, GridType::Collocated)), +#if defined(WARPX_DIM_3D) + modified_ky_vec_centered(spectral_kspace.getModifiedKComponent(dm, 1, norder_y, GridType::Collocated)), + modified_kz_vec_centered(spectral_kspace.getModifiedKComponent(dm, 2, norder_z, GridType::Collocated)), +#else + modified_kz_vec_centered(spectral_kspace.getModifiedKComponent(dm, 1, norder_z, GridType::Collocated)), +#endif + m_v_galilean(v_galilean), + m_dt(dt), + m_dive_cleaning(dive_cleaning), + m_divb_cleaning(divb_cleaning) { const BoxArray& ba = spectral_kspace.spectralspace_ba; - // Allocate the arrays of coefficients - C_coef = SpectralRealCoefficients(ba, dm, 1, 0); - S_ck_coef = SpectralRealCoefficients(ba, dm, 1, 0); + m_is_galilean = (v_galilean[0] != 0.) || (v_galilean[1] != 0.) || (v_galilean[2] != 0.); + + // Allocate arrays of coefficients + C_coef = SpectralRealCoefficients(ba, dm, 1, 0); + S_ck_coef = SpectralRealCoefficients(ba, dm, 1, 0); inv_k2_coef = SpectralRealCoefficients(ba, dm, 1, 0); - InitializeSpectralCoefficients(spectral_kspace, dm, dt); -} + // Allocate this coefficient only with Galilean PSATD + if (m_is_galilean) + { + T2_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + } -/* Advance the E and B field in spectral space (stored in `f`) - * over one time step */ -void -PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { + InitializeSpectralCoefficients(spectral_kspace, dm); +} +void PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const +{ const bool dive_cleaning = m_dive_cleaning; const bool divb_cleaning = m_divb_cleaning; + const bool is_galilean = m_is_galilean; const SpectralFieldIndex& Idx = m_spectral_index; // Loop over boxes - for (MFIter mfi(f.fields); mfi.isValid(); ++mfi){ - - const Box& bx = f.fields[mfi].box(); + for (amrex::MFIter mfi(f.fields); mfi.isValid(); ++mfi) + { + const amrex::Box& bx = f.fields[mfi].box(); // Extract arrays for the fields to be updated - const Array4 fields = f.fields[mfi].array(); + const amrex::Array4 fields = f.fields[mfi].array(); // Extract arrays for the coefficients - const Array4 C_arr = C_coef[mfi].array(); - const Array4 S_ck_arr = S_ck_coef[mfi].array(); - const Array4 inv_k2_arr = inv_k2_coef[mfi].array(); + const amrex::Array4 C_arr = C_coef[mfi].array(); + const amrex::Array4 S_ck_arr = S_ck_coef[mfi].array(); + const amrex::Array4 inv_k2_arr = inv_k2_coef[mfi].array(); + + amrex::Array4 T2_arr; + if (is_galilean) + { + T2_arr = T2_coef[mfi].array(); + } // Extract pointers for the k vectors - const Real* modified_kx_arr = modified_kx_vec[mfi].dataPtr(); + const amrex::Real* modified_kx_arr = modified_kx_vec[mfi].dataPtr(); #if defined(WARPX_DIM_3D) - const Real* modified_ky_arr = modified_ky_vec[mfi].dataPtr(); + const amrex::Real* modified_ky_arr = modified_ky_vec[mfi].dataPtr(); #endif - const Real* modified_kz_arr = modified_kz_vec[mfi].dataPtr(); + const amrex::Real* modified_kz_arr = modified_kz_vec[mfi].dataPtr(); const amrex::Real dt = m_dt; @@ -154,28 +181,29 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { } // k vector values, and coefficients - const Real kx = modified_kx_arr[i]; + const amrex::Real kx = modified_kx_arr[i]; #if defined(WARPX_DIM_3D) - const Real ky = modified_ky_arr[j]; - const Real kz = modified_kz_arr[k]; + const amrex::Real ky = modified_ky_arr[j]; + const amrex::Real kz = modified_kz_arr[k]; #else - constexpr Real ky = 0._rt; - const Real kz = modified_kz_arr[j]; + constexpr amrex::Real ky = 0._rt; + const amrex::Real kz = modified_kz_arr[j]; #endif - constexpr Real c2 = PhysConst::c * PhysConst::c; + constexpr amrex::Real c2 = PhysConst::c*PhysConst::c; const Complex I = Complex{0._rt, 1._rt}; - const Real kx2 = kx*kx; - const Real ky2 = ky*ky; - const Real kz2 = kz*kz; - const Real k_norm = std::sqrt(kx2 + ky2 + kz2); - - if (k_norm != 0._rt) { + const amrex::Real kx2 = kx*kx; + const amrex::Real ky2 = ky*ky; + const amrex::Real kz2 = kz*kz; + const amrex::Real knorm = std::sqrt(kx2 + ky2 + kz2); + if (knorm != 0._rt) + { const amrex::Real C = C_arr(i,j,k); const amrex::Real S_ck = S_ck_arr(i,j,k); const amrex::Real inv_k2 = inv_k2_arr(i,j,k); + const Complex T2 = (is_galilean) ? T2_arr(i,j,k) : 1.0_rt; const amrex::Real C1 = (kx2 * C + ky2 + kz2) * inv_k2; const amrex::Real C2 = (kx2 + ky2 * C + kz2) * inv_k2; @@ -218,42 +246,42 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { const Complex C22_c2 = C22 / c2; // Update E - fields(i,j,k,Idx.Exy) = C2 * Exy + C5 * Exz + C9 * Ey - + C10 * Bx + C11 * By + C19 * Bz; + fields(i,j,k,Idx.Exy) = T2 * (C2 * Exy + C5 * Exz + C9 * Ey + + C10 * Bx + C11 * By + C19 * Bz); - fields(i,j,k,Idx.Exz) = C6 * Exy + C3 * Exz + C8 * Ez - - C10 * Bx - C22 * By - C12 * Bz; + fields(i,j,k,Idx.Exz) = T2 * (C6 * Exy + C3 * Exz + C8 * Ez + - C10 * Bx - C22 * By - C12 * Bz); - fields(i,j,k,Idx.Eyz) = C3 * Eyz + C6 * Eyx + C7 * Ez - + C21 * Bx + C10 * By + C13 * Bz; + fields(i,j,k,Idx.Eyz) = T2 * (C3 * Eyz + C6 * Eyx + C7 * Ez + + C21 * Bx + C10 * By + C13 * Bz); - fields(i,j,k,Idx.Eyx) = C9 * Ex + C4 * Eyz + C1 * Eyx - - C14 * Bx - C10 * By - C18 * Bz; + fields(i,j,k,Idx.Eyx) = T2 * (C9 * Ex + C4 * Eyz + C1 * Eyx + - C14 * Bx - C10 * By - C18 * Bz); - fields(i,j,k,Idx.Ezx) = C8 * Ex + C1 * Ezx + C4 * Ezy - + C15 * Bx + C17 * By + C10 * Bz; + fields(i,j,k,Idx.Ezx) = T2 * (C8 * Ex + C1 * Ezx + C4 * Ezy + + C15 * Bx + C17 * By + C10 * Bz); - fields(i,j,k,Idx.Ezy) = C7 * Ey + C5 * Ezx + C2 * Ezy - - C20 * Bx - C16 * By - C10 * Bz; + fields(i,j,k,Idx.Ezy) = T2 * (C7 * Ey + C5 * Ezx + C2 * Ezy + - C20 * Bx - C16 * By - C10 * Bz); // Update B - fields(i,j,k,Idx.Bxy) = C2 * Bxy + C5 * Bxz + C9 * By - - C10_c2 * Ex - C11_c2 * Ey - C19_c2 * Ez; + fields(i,j,k,Idx.Bxy) = T2 * (C2 * Bxy + C5 * Bxz + C9 * By + - C10_c2 * Ex - C11_c2 * Ey - C19_c2 * Ez); - fields(i,j,k,Idx.Bxz) = C6 * Bxy + C3 * Bxz + C8 * Bz - + C10_c2 * Ex + C22_c2 * Ey + C12_c2 * Ez; + fields(i,j,k,Idx.Bxz) = T2 * (C6 * Bxy + C3 * Bxz + C8 * Bz + + C10_c2 * Ex + C22_c2 * Ey + C12_c2 * Ez); - fields(i,j,k,Idx.Byz) = C3 * Byz + C6 * Byx + C7 * Bz - - C21_c2 * Ex - C10_c2 * Ey - C13_c2 * Ez; + fields(i,j,k,Idx.Byz) = T2 * (C3 * Byz + C6 * Byx + C7 * Bz + - C21_c2 * Ex - C10_c2 * Ey - C13_c2 * Ez); - fields(i,j,k,Idx.Byx) = C9 * Bx + C4 * Byz + C1 * Byx - + C14_c2 * Ex + C10_c2 * Ey + C18_c2 * Ez; + fields(i,j,k,Idx.Byx) = T2 * (C9 * Bx + C4 * Byz + C1 * Byx + + C14_c2 * Ex + C10_c2 * Ey + C18_c2 * Ez); - fields(i,j,k,Idx.Bzx) = C8 * Bx + C1 * Bzx + C4 * Bzy - - C15_c2 * Ex - C17_c2 * Ey - C10_c2 * Ez; + fields(i,j,k,Idx.Bzx) = T2 * (C8 * Bx + C1 * Bzx + C4 * Bzy + - C15_c2 * Ex - C17_c2 * Ey - C10_c2 * Ez); - fields(i,j,k,Idx.Bzy) = C7 * By + C5 * Bzx + C2 * Bzy - + C20_c2 * Ex + C16_c2 * Ey + C10_c2 * Ez; + fields(i,j,k,Idx.Bzy) = T2 * (C7 * By + C5 * Bzx + C2 * Bzy + + C20_c2 * Ex + C16_c2 * Ey + C10_c2 * Ez); } else if (dive_cleaning && divb_cleaning) { @@ -266,80 +294,80 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { const Complex C25_c2 = C25 / c2; // Update E - fields(i,j,k,Idx.Exx) = C1 * Exx + C4 * Exy + C4 * Exz - - C9 * Ey - C8 * Ez + C23 * F; + fields(i,j,k,Idx.Exx) = T2 * (C1 * Exx + C4 * Exy + C4 * Exz + - C9 * Ey - C8 * Ez + C23 * F); - fields(i,j,k,Idx.Exy) = C5 * Exx + C2 * Exy + C5 * Exz - + C9 * Ey + C24 * Bz - C7 * G; + fields(i,j,k,Idx.Exy) = T2 * (C5 * Exx + C2 * Exy + C5 * Exz + + C9 * Ey + C24 * Bz - C7 * G); - fields(i,j,k,Idx.Exz) = C6 * Exx + C6 * Exy + C3 * Exz - + C8 * Ez - C25 * By + C7 * G; + fields(i,j,k,Idx.Exz) = T2 * (C6 * Exx + C6 * Exy + C3 * Exz + + C8 * Ez - C25 * By + C7 * G); - fields(i,j,k,Idx.Eyx) = C9 * Ex + C1 * Eyx + C4 * Eyy - + C4 * Eyz - C23 * Bz + C8 * G; + fields(i,j,k,Idx.Eyx) = T2 * (C9 * Ex + C1 * Eyx + C4 * Eyy + + C4 * Eyz - C23 * Bz + C8 * G); - fields(i,j,k,Idx.Eyy) = - C9 * Ex + C5 * Eyx + C2 * Eyy - + C5 * Eyz - C7 * Ez + C24 * F; + fields(i,j,k,Idx.Eyy) = T2 * (- C9 * Ex + C5 * Eyx + C2 * Eyy + + C5 * Eyz - C7 * Ez + C24 * F); - fields(i,j,k,Idx.Eyz) = C6 * Eyx + C6 * Eyy + C3 * Eyz - + C7 * Ez + C25 * Bx - C8 * G; + fields(i,j,k,Idx.Eyz) = T2 * (C6 * Eyx + C6 * Eyy + C3 * Eyz + + C7 * Ez + C25 * Bx - C8 * G); - fields(i,j,k,Idx.Ezx) = C8 * Ex + C1 * Ezx + C4 * Ezy - + C4 * Ezz + C23 * By - C9 * G; + fields(i,j,k,Idx.Ezx) = T2 * (C8 * Ex + C1 * Ezx + C4 * Ezy + + C4 * Ezz + C23 * By - C9 * G); - fields(i,j,k,Idx.Ezy) = C7 * Ey + C5 * Ezx + C2 * Ezy - + C5 * Ezz - C24 * Bx + C9 * G; + fields(i,j,k,Idx.Ezy) = T2 * (C7 * Ey + C5 * Ezx + C2 * Ezy + + C5 * Ezz - C24 * Bx + C9 * G); - fields(i,j,k,Idx.Ezz) = - C8 * Ex - C7 * Ey + C6 * Ezx - + C6 * Ezy + C3 * Ezz + C25 * F; + fields(i,j,k,Idx.Ezz) = T2 * (- C8 * Ex - C7 * Ey + C6 * Ezx + + C6 * Ezy + C3 * Ezz + C25 * F); // Update B - fields(i,j,k,Idx.Bxx) = C1 * Bxx + C4 * Bxy + C4 * Bxz - - C9 * By - C8 * Bz + C23_c2 * G; + fields(i,j,k,Idx.Bxx) = T2 * (C1 * Bxx + C4 * Bxy + C4 * Bxz + - C9 * By - C8 * Bz + C23_c2 * G); - fields(i,j,k,Idx.Bxy) = - C24_c2 * Ez + C5 * Bxx + C2 * Bxy - + C5 * Bxz + C9 * By + C7 * F; + fields(i,j,k,Idx.Bxy) = T2 * (- C24_c2 * Ez + C5 * Bxx + C2 * Bxy + + C5 * Bxz + C9 * By + C7 * F); - fields(i,j,k,Idx.Bxz) = C25_c2 * Ey + C6 * Bxx + C6 * Bxy - + C3 * Bxz + C8 * Bz - C7 * F; + fields(i,j,k,Idx.Bxz) = T2 * (C25_c2 * Ey + C6 * Bxx + C6 * Bxy + + C3 * Bxz + C8 * Bz - C7 * F); - fields(i,j,k,Idx.Byx) = C23_c2 * Ez + C9 * Bx + C1 * Byx - + C4 * Byy + C4 * Byz - C8 * F; + fields(i,j,k,Idx.Byx) = T2 * (C23_c2 * Ez + C9 * Bx + C1 * Byx + + C4 * Byy + C4 * Byz - C8 * F); - fields(i,j,k,Idx.Byy) = - C9 * Bx + C5 * Byx + C2 * Byy - + C5 * Byz - C7 * Bz + C24_c2 * G; + fields(i,j,k,Idx.Byy) = T2 * (- C9 * Bx + C5 * Byx + C2 * Byy + + C5 * Byz - C7 * Bz + C24_c2 * G); - fields(i,j,k,Idx.Byz) = - C25_c2 * Ex + C6 * Byx + C6 * Byy - + C3 * Byz + C7 * Bz + C8 * F; + fields(i,j,k,Idx.Byz) = T2 * (- C25_c2 * Ex + C6 * Byx + C6 * Byy + + C3 * Byz + C7 * Bz + C8 * F); - fields(i,j,k,Idx.Bzx) = - C23_c2 * Ey + C8 * Bx + C1 * Bzx - + C4 * Bzy + C4 * Bzz + C9 * F; + fields(i,j,k,Idx.Bzx) = T2 * (- C23_c2 * Ey + C8 * Bx + C1 * Bzx + + C4 * Bzy + C4 * Bzz + C9 * F); - fields(i,j,k,Idx.Bzy) = C24_c2 * Ex + C7 * By + C5 * Bzx - + C2 * Bzy + C5 * Bzz - C9 * F; + fields(i,j,k,Idx.Bzy) = T2 * (C24_c2 * Ex + C7 * By + C5 * Bzx + + C2 * Bzy + C5 * Bzz - C9 * F); - fields(i,j,k,Idx.Bzz) = - C8 * Bx - C7 * By + C6 * Bzx - + C6 * Bzy + C3 * Bzz + C25_c2 * G; + fields(i,j,k,Idx.Bzz) = T2 * (- C8 * Bx - C7 * By + C6 * Bzx + + C6 * Bzy + C3 * Bzz + C25_c2 * G); // Update F - fields(i,j,k,Idx.Fx) = C23_c2 * Ex + C8 * By - C9 * Bz - + C1 * Fx + C4 * Fy + C4 * Fz; + fields(i,j,k,Idx.Fx) = T2 * (C23_c2 * Ex + C8 * By - C9 * Bz + + C1 * Fx + C4 * Fy + C4 * Fz); - fields(i,j,k,Idx.Fy) = C24_c2 * Ey - C7 * Bx + C9 * Bz - + C5 * Fx + C2 * Fy + C5 * Fz; + fields(i,j,k,Idx.Fy) = T2 * (C24_c2 * Ey - C7 * Bx + C9 * Bz + + C5 * Fx + C2 * Fy + C5 * Fz); - fields(i,j,k,Idx.Fz) = C25_c2 * Ez + C7 * Bx - C8 * By - + C6 * Fx + C6 * Fy + C3 * Fz; + fields(i,j,k,Idx.Fz) = T2 * (C25_c2 * Ez + C7 * Bx - C8 * By + + C6 * Fx + C6 * Fy + C3 * Fz); // Update G - fields(i,j,k,Idx.Gx) = - C8 * Ey + C9 * Ez + C23 * Bx - + C1 * Gx + C4 * Gy + C4 * Gz; + fields(i,j,k,Idx.Gx) = T2 * (- C8 * Ey + C9 * Ez + C23 * Bx + + C1 * Gx + C4 * Gy + C4 * Gz); - fields(i,j,k,Idx.Gy) = C7 * Ex - C9 * Ez + C24 * By - + C5 * Gx + C2 * Gy + C5 * Gz; + fields(i,j,k,Idx.Gy) = T2 * (C7 * Ex - C9 * Ez + C24 * By + + C5 * Gx + C2 * Gy + C5 * Gz); - fields(i,j,k,Idx.Gz) = - C7 * Ex + C8 * Ey + C25 * Bz - + C6 * Gx + C6 * Gy + C3 * Gz; + fields(i,j,k,Idx.Gz) = T2 * (- C7 * Ex + C8 * Ey + C25 * Bz + + C6 * Gx + C6 * Gy + C3 * Gz); } } }); @@ -348,71 +376,95 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { void PsatdAlgorithmPml::InitializeSpectralCoefficients ( const SpectralKSpace& spectral_kspace, - const amrex::DistributionMapping& dm, - const amrex::Real dt) + const amrex::DistributionMapping& dm) { - const BoxArray& ba = spectral_kspace.spectralspace_ba; + const amrex::Real dt = m_dt; + const bool is_galilean = m_is_galilean; - // Fill them with the right values: - // Loop over boxes and allocate the corresponding coefficients - // for each box owned by the local MPI proc - for (MFIter mfi(ba, dm); mfi.isValid(); ++mfi) { + const amrex::BoxArray& ba = spectral_kspace.spectralspace_ba; - const Box& bx = ba[mfi]; + // Loop over boxes and allocate the corresponding coefficients + // for each box owned by the local MPI process + for (amrex::MFIter mfi(ba, dm); mfi.isValid(); ++mfi) + { + const amrex::Box& bx = ba[mfi]; // Extract pointers for the k vectors - const Real* modified_kx = modified_kx_vec[mfi].dataPtr(); + const amrex::Real* kx = modified_kx_vec[mfi].dataPtr(); + const amrex::Real* kx_c = modified_kx_vec_centered[mfi].dataPtr(); #if defined(WARPX_DIM_3D) - const Real* modified_ky = modified_ky_vec[mfi].dataPtr(); + const amrex::Real* ky = modified_ky_vec[mfi].dataPtr(); + const amrex::Real* ky_c = modified_ky_vec_centered[mfi].dataPtr(); #endif - const Real* modified_kz = modified_kz_vec[mfi].dataPtr(); + const amrex::Real* kz = modified_kz_vec[mfi].dataPtr(); + const amrex::Real* kz_c = modified_kz_vec_centered[mfi].dataPtr(); // Extract arrays for the coefficients - const Array4 C = C_coef[mfi].array(); - const Array4 S_ck = S_ck_coef[mfi].array(); - const Array4 inv_k2 = inv_k2_coef[mfi].array(); + const amrex::Array4 C = C_coef[mfi].array(); + const amrex::Array4 S_ck = S_ck_coef[mfi].array(); + const amrex::Array4 inv_k2 = inv_k2_coef[mfi].array(); - // Loop over indices within one box - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + amrex::Array4 T2; + if (is_galilean) { - const Real kx = modified_kx[i]; + T2 = T2_coef[mfi].array(); + } + + // Extract Galilean velocity + amrex::Real vg_x = m_v_galilean[0]; #if defined(WARPX_DIM_3D) - const Real ky = modified_ky[j]; - const Real kz = modified_kz[k]; -#else - constexpr Real ky = 0._rt; - const Real kz = modified_kz[j]; + amrex::Real vg_y = m_v_galilean[1]; #endif + amrex::Real vg_z = m_v_galilean[2]; + // Loop over indices within one box + ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { // Calculate norm of vector - const Real k_norm = std::sqrt(kx*kx + ky*ky + kz*kz); - const Real k2 = k_norm * k_norm; + const amrex::Real knorm = std::sqrt( + amrex::Math::powi<2>(kx[i]) + +#if (AMREX_SPACEDIM==3) + amrex::Math::powi<2>(ky[j]) + amrex::Math::powi<2>(kz[k])); +#else + amrex::Math::powi<2>(kz[j])); +#endif + // Calculate the dot product of the k vector with the Galilean velocity. + // This has to be computed always with the centered (collocated) finite-order + // modified k vectors, to work correctly for both collocated and staggered grids. + // w_c = 0 always with standard PSATD (zero Galilean velocity). + const amrex::Real w_c = kx_c[i]*vg_x + +#if (AMREX_SPACEDIM==3) + ky_c[j]*vg_y + kz_c[k]*vg_z; +#else + kz_c[j]*vg_z; +#endif + constexpr amrex::Real c = PhysConst::c; + constexpr Complex I{0._rt,1._rt}; - // Calculate coefficients - constexpr Real c = PhysConst::c; + // Coefficients for knorm = 0 do not need to be set + if (knorm != 0._rt) + { + C(i,j,k) = std::cos(c*knorm*dt); + S_ck(i,j,k) = std::sin(c*knorm*dt) / (c*knorm); + inv_k2(i,j,k) = 1._rt / (knorm*knorm); - // Coefficients for k_norm = 0 do not need to be set - if (k_norm != 0._rt) { - C(i,j,k) = std::cos(c * k_norm * dt); - S_ck(i,j,k) = std::sin(c * k_norm * dt) / (c * k_norm); - inv_k2(i,j,k) = 1._rt / k2; + if (is_galilean) + { + T2(i,j,k) = amrex::exp(I*w_c*dt); + } } }); } } -void -PsatdAlgorithmPml::CurrentCorrection (SpectralFieldData& /*field_data*/) +void PsatdAlgorithmPml::CurrentCorrection (SpectralFieldData& /*field_data*/) { - WARPX_ABORT_WITH_MESSAGE( - "Current correction not implemented for PML PSATD"); + WARPX_ABORT_WITH_MESSAGE("Current correction not implemented for PML PSATD"); } -void -PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) +void PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) { - WARPX_ABORT_WITH_MESSAGE( - "Vay deposition not implemented for PML PSATD"); + WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented for PML PSATD"); } #endif // WARPX_USE_PSATD diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H index 64ffc80ac37..2285ab05679 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H @@ -32,14 +32,11 @@ ! (www.jpmoreau.fr) ! ------------------------------------------------------------------------ */ -#include "Utils/WarpXConst.H" +#ifndef WARPX_BESSEL_ROOTS_H_ +#define WARPX_BESSEL_ROOTS_H_ #include - -#include - - -void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int *ier); +#include /*---------------------------------------------------------------------- * calculate the first nk zeroes of bessel function j(n, x) @@ -56,112 +53,6 @@ void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int * abramowitz m. & stegun irene a. * handbook of mathematical functions */ -void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vector &ier) { - using namespace amrex::literals; - - amrex::Real zeroj; - int ierror, ik, k; - - const amrex::Real tol = 1e-14_rt; - const amrex::Real nitmx = 10; - - const amrex::Real c1 = 1.8557571_rt; - const amrex::Real c2 = 1.033150_rt; - const amrex::Real c3 = 0.00397_rt; - const amrex::Real c4 = 0.0908_rt; - const amrex::Real c5 = 0.043_rt; - - const amrex::Real t0 = 4.0_rt*n*n; - const amrex::Real t1 = t0 - 1.0_rt; - const amrex::Real t3 = 4.0_rt*t1*(7.0_rt*t0 - 31.0_rt); - const amrex::Real t5 = 32.0_rt*t1*((83.0_rt*t0 - 982.0_rt)*t0 + 3779.0_rt); - const amrex::Real t7 = 64.0_rt*t1*(((6949.0_rt*t0 - 153855.0_rt)*t0 + 1585743.0_rt)*t0 - 6277237.0_rt); - - roots.resize(nk); - ier.resize(nk); - - // first zero - if (n == 0) { - zeroj = c1 + c2 - c3 - c4 + c5; - SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); - ier[0] = ierror; - roots[0] = zeroj; - ik = 1; - } - else { - // Include the trivial root - ier[0] = 0; - roots[0] = 0.; - const amrex::Real f1 = std::pow(n, (1.0_rt/3.0_rt)); - const amrex::Real f2 = f1*f1*n; - const amrex::Real f3 = f1*n*n; - zeroj = n + c1*f1 + (c2/f1) - (c3/n) - (c4/f2) + (c5/f3); - SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); - ier[1] = ierror; - roots[1] = zeroj; - ik = 2; - } - - // other zeroes - // k counts the nontrivial roots - // ik counts all roots - k = 2; - while (ik < nk) { - - // mac mahon's series for k >> n - const amrex::Real b0 = (k + 0.5_rt*n - 0.25_rt)*MathConst::pi; - const amrex::Real b1 = 8.0_rt*b0; - const amrex::Real b2 = b1*b1; - const amrex::Real b3 = 3.0_rt*b1*b2; - const amrex::Real b5 = 5.0_rt*b3*b2; - const amrex::Real b7 = 7.0_rt*b5*b2; - - zeroj = b0 - (t1/b1) - (t3/b3) - (t5/b5) - (t7/b7); - - const amrex::Real errj = std::abs(jn(n, zeroj)); - - // improve solution using procedure SecantRootFinder - if (errj > tol) SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); - - roots[ik] = zeroj; - ier[ik] = ierror; - - k++; - ik++; - } -} - -void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int *ier) { - using namespace amrex::literals; - - amrex::Real p0, p1, q0, q1, dp, p; - amrex::Real c[2]; - - c[0] = 0.95_rt; - c[1] = 0.999_rt; - *ier = 0; - - p = *zeroj; - for (int ntry=0 ; ntry <= 1 ; ntry++) { - p0 = c[ntry]*(*zeroj); +void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vector &ier); - p1 = *zeroj; - q0 = jn(n, p0); - q1 = jn(n, p1); - for (int it=1; it <= nitmx; it++) { - if (q1 == q0) break; - p = p1 - q1*(p1 - p0)/(q1 - q0); - dp = p - p1; - if (it > 1 && std::abs(dp) < tol) { - *zeroj = p; - return; - } - p0 = p1; - q0 = q1; - p1 = p; - q1 = jn(n, p1); - } - } - *ier = 3; - *zeroj = p; -} +#endif // WARPX_BESSEL_ROOTS_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp new file mode 100644 index 00000000000..210a4baffc7 --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp @@ -0,0 +1,153 @@ +/* Copyright 2019 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +/* ------------------------------------------------------------------------- +! program to calculate the first zeroes (root abscissas) of the first +! kind bessel function of integer order n using the subroutine rootj. +! -------------------------------------------------------------------------- +! sample run: +! +! (calculate the first 10 zeroes of 1st kind bessel function of order 2). +! +! zeroes of bessel function of order: 2 +! +! number of calculated zeroes: 10 +! +! table of root abcissas (5 items per line) +! 5.135622 8.417244 11.619841 14.795952 17.959819 + 21.116997 24.270112 27.420574 30.569204 33.716520 +! +! table of error codes (5 items per line) +! 0 0 0 0 0 +! 0 0 0 0 0 +! +! -------------------------------------------------------------------------- +! reference: from numath library by tuan dang trong in fortran 77 +! [bibli 18]. +! +! c++ release 1.0 by j-p moreau, paris +! (www.jpmoreau.fr) +! ------------------------------------------------------------------------ */ + +#include "BesselRoots.H" + +#include "Utils/WarpXConst.H" + +#include + +namespace{ + + void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int *ier) { + using namespace amrex::literals; + + amrex::Real p0, p1, q0, q1, dp, p; + amrex::Real c[2]; + + c[0] = 0.95_rt; + c[1] = 0.999_rt; + *ier = 0; + + p = *zeroj; + for (int ntry=0 ; ntry <= 1 ; ntry++) { + p0 = c[ntry]*(*zeroj); + + p1 = *zeroj; + q0 = jn(n, p0); + q1 = jn(n, p1); + for (int it=1; it <= nitmx; it++) { + if (q1 == q0) break; + p = p1 - q1*(p1 - p0)/(q1 - q0); + dp = p - p1; + if (it > 1 && std::abs(dp) < tol) { + *zeroj = p; + return; + } + p0 = p1; + q0 = q1; + p1 = p; + q1 = jn(n, p1); + } + } + *ier = 3; + *zeroj = p; + } + +} + +void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vector &ier) { + using namespace amrex::literals; + + amrex::Real zeroj; + int ierror, ik, k; + + const amrex::Real tol = 1e-14_rt; + const amrex::Real nitmx = 10; + + const amrex::Real c1 = 1.8557571_rt; + const amrex::Real c2 = 1.033150_rt; + const amrex::Real c3 = 0.00397_rt; + const amrex::Real c4 = 0.0908_rt; + const amrex::Real c5 = 0.043_rt; + + const amrex::Real t0 = 4.0_rt*n*n; + const amrex::Real t1 = t0 - 1.0_rt; + const amrex::Real t3 = 4.0_rt*t1*(7.0_rt*t0 - 31.0_rt); + const amrex::Real t5 = 32.0_rt*t1*((83.0_rt*t0 - 982.0_rt)*t0 + 3779.0_rt); + const amrex::Real t7 = 64.0_rt*t1*(((6949.0_rt*t0 - 153855.0_rt)*t0 + 1585743.0_rt)*t0 - 6277237.0_rt); + + roots.resize(nk); + ier.resize(nk); + + // first zero + if (n == 0) { + zeroj = c1 + c2 - c3 - c4 + c5; + ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + ier[0] = ierror; + roots[0] = zeroj; + ik = 1; + } + else { + // Include the trivial root + ier[0] = 0; + roots[0] = 0.; + const amrex::Real f1 = std::pow(n, (1.0_rt/3.0_rt)); + const amrex::Real f2 = f1*f1*n; + const amrex::Real f3 = f1*n*n; + zeroj = n + c1*f1 + (c2/f1) - (c3/n) - (c4/f2) + (c5/f3); + ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + ier[1] = ierror; + roots[1] = zeroj; + ik = 2; + } + + // other zeroes + // k counts the nontrivial roots + // ik counts all roots + k = 2; + while (ik < nk) { + + // mac mahon's series for k >> n + const amrex::Real b0 = (k + 0.5_rt*n - 0.25_rt)*MathConst::pi; + const amrex::Real b1 = 8.0_rt*b0; + const amrex::Real b2 = b1*b1; + const amrex::Real b3 = 3.0_rt*b1*b2; + const amrex::Real b5 = 5.0_rt*b3*b2; + const amrex::Real b7 = 7.0_rt*b5*b2; + + zeroj = b0 - (t1/b1) - (t3/b3) - (t5/b5) - (t7/b7); + + const amrex::Real errj = std::abs(jn(n, zeroj)); + + // improve solution using procedure SecantRootFinder + if (errj > tol) ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + + roots[ik] = zeroj; + ier[ik] = ierror; + + k++; + ik++; + } +} diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt index a3d2b60bcc4..70c9740367f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt @@ -1,5 +1,6 @@ target_sources(lib_rz PRIVATE + BesselRoots.cpp SpectralHankelTransformer.cpp HankelTransform.cpp ) diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package index 8bb1d7ef7b4..37ca9a93186 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package @@ -1,3 +1,4 @@ +CEXE_sources += BesselRoots.cpp CEXE_sources += SpectralHankelTransformer.cpp CEXE_sources += HankelTransform.cpp diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 362af06a354..80fb2c39545 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -52,11 +52,11 @@ SpectralSolver::SpectralSolver( // - Select the algorithm depending on the input parameters // Initialize the corresponding coefficients over k space - if (pml) // PSATD equations in the PML region + if (pml) // PSATD or Galilean PSATD equations in the PML region { algorithm = std::make_unique( k_space, dm, m_spectral_index, norder_x, norder_y, norder_z, grid_type, - dt, dive_cleaning, divb_cleaning); + v_galilean, dt, dive_cleaning, divb_cleaning); } else // PSATD equations in the regular domain { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index b502620641b..36029e7d3eb 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -10,6 +10,8 @@ #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" @@ -30,6 +32,13 @@ void WarpX::HybridPICEvolveFields () // Perform current deposition at t_{n+1/2}. mypc->DepositCurrent(current_fp, dt[0], -0.5_rt * dt[0]); + // Deposit cold-relativistic fluid charge and current + if (do_fluid_species) { + int const lev = 0; + myfl->DepositCharge(lev, *rho_fp[lev]); + myfl->DepositCurrent(lev, *current_fp[lev][0], *current_fp[lev][1], *current_fp[lev][2]); + } + // Synchronize J and rho: // filter (if used), exchange guard cells, interpolate across MR levels // and apply boundary conditions diff --git a/Source/Fluids/CMakeLists.txt b/Source/Fluids/CMakeLists.txt new file mode 100644 index 00000000000..a5f28debbbd --- /dev/null +++ b/Source/Fluids/CMakeLists.txt @@ -0,0 +1,8 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + MultiFluidContainer.cpp + WarpXFluidContainer.cpp + ) +endforeach() diff --git a/Source/Fluids/Make.package b/Source/Fluids/Make.package new file mode 100644 index 00000000000..96bce415c4a --- /dev/null +++ b/Source/Fluids/Make.package @@ -0,0 +1,4 @@ +CEXE_sources += MultiFluidContainer.cpp +CEXE_sources += WarpXFluidContainer.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Fluids diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H new file mode 100644 index 00000000000..94d95d179e5 --- /dev/null +++ b/Source/Fluids/MultiFluidContainer.H @@ -0,0 +1,78 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_MultiFluidContainer_H_ +#define WARPX_MultiFluidContainer_H_ +#include "Evolve/WarpXDtType.H" + +#include "WarpXFluidContainer_fwd.H" + +#include +#include + +#include + +/** + * The class MultiFluidContainer holds multiple instances of the + * class WarpXFluidContainer, stored in its member variable "allcontainers". + * The class WarpX typically has a single (pointer to an) instance of + * MultiFluidContainer. + * + * MultiFluidContainer typically has two types of functions: + * - Functions that loop over all instances of WarpXFluidContainer in + * allcontainers and calls the corresponding function (for instance, + * MultiFluidContainer::Evolve loops over all fluid containers and + * calls the corresponding WarpXFluidContainer::Evolve function). + * - Functions that specifically handle multiple species (for instance + * ReadParameters). + */ +class MultiFluidContainer +{ + +public: + + MultiFluidContainer (int nlevs_max); + + ~MultiFluidContainer() {} + + WarpXFluidContainer& + GetFluidContainer (int ispecies) const {return *allcontainers[ispecies];} + +#ifdef WARPX_USE_OPENPMD + std::unique_ptr& GetUniqueContainer(int ispecies) { + return allcontainers[ispecies]; + } +#endif + + void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + + void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + + /// + /// This evolves all the fluids by one PIC time step, including current deposition, the + /// field solve, and pushing the fluids, for all the species in the MultiFluidContainer. + /// + void Evolve (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + amrex::Real cur_time, bool skip_deposition=false); + + int nSpecies() const {return species_names.size();} + + void DepositCharge (int lev, amrex::MultiFab &rho); + void DepositCurrent (int lev, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + +private: + + std::vector species_names; + + // Vector of fluid species + amrex::Vector> allcontainers; + +}; +#endif /*WARPX_MultiFluidContainer_H_*/ diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp new file mode 100644 index 00000000000..234cefb4f07 --- /dev/null +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -0,0 +1,73 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" +#include "Utils/Parser/ParserUtils.H" + +#include + +using namespace amrex; + +MultiFluidContainer::MultiFluidContainer (int nlevs_max) +{ + const ParmParse pp_fluids("fluids"); + pp_fluids.queryarr("species_names", species_names); + + const int nspecies = static_cast(species_names.size()); + + allcontainers.resize(nspecies); + for (int i = 0; i < nspecies; ++i) { + allcontainers[i] = std::make_unique(nlevs_max, i, species_names[i]); + } +} + +void +MultiFluidContainer::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm) +{ + for (auto& fl : allcontainers) { + fl->AllocateLevelMFs(lev, ba, dm); + } +} + +void +MultiFluidContainer::InitData (int lev, amrex::Box init_box, amrex::Real cur_time) +{ + for (auto& fl : allcontainers) { + fl->InitData(lev, init_box, cur_time); + } +} + + +void +MultiFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho) +{ + for (auto& fl : allcontainers) { + fl->DepositCharge(lev,rho); + } +} + +void +MultiFluidContainer::DepositCurrent (int lev, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz) +{ + for (auto& fl : allcontainers) { + fl->DepositCurrent(lev,jx,jy,jz); + } +} + +void +MultiFluidContainer::Evolve (int lev, + const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, + const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + MultiFab* rho, MultiFab& jx, MultiFab& jy, MultiFab& jz, + amrex::Real cur_time, bool skip_deposition) +{ + for (auto& fl : allcontainers) { + fl->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, rho, jx, jy, jz, cur_time, skip_deposition); + } +} diff --git a/Source/Fluids/MultiFluidContainer_fwd.H b/Source/Fluids/MultiFluidContainer_fwd.H new file mode 100644 index 00000000000..a1a55c3b450 --- /dev/null +++ b/Source/Fluids/MultiFluidContainer_fwd.H @@ -0,0 +1,12 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_MultiFluidContainer_fwd_H_ + +class MultiFluidContainer; + +#endif /* WARPX_MultiFluidContainer_fwd_H_ */ diff --git a/Source/Fluids/MusclHancockUtils.H b/Source/Fluids/MusclHancockUtils.H new file mode 100644 index 00000000000..d0c37f9c58f --- /dev/null +++ b/Source/Fluids/MusclHancockUtils.H @@ -0,0 +1,484 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_MusclHancock_H_ +#define WARPX_MusclHancock_H_ + +#include +#include +#include +#include + + +// Euler push for momentum source (r-direction) +// Note: assumes U normalized by c +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real F_r (amrex::Real r, amrex::Real u_r, amrex::Real u_theta, amrex::Real u_z, amrex::Real dt) +{ + return dt*(-u_theta*u_theta/r)/sqrt(1.0 + u_r*u_r + u_theta*u_theta + u_z*u_z) + u_r; +} + +// Euler push for momentum source (theta-direction) +// Note: assumes U normalized by c +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real F_theta (amrex::Real r, amrex::Real u_r, amrex::Real u_theta, amrex::Real u_z, amrex::Real dt) +{ + return dt*(u_theta*u_r/r)/sqrt(1.0 + u_r*u_r + u_theta*u_theta + u_z*u_z) + u_theta; +} +// Velocity at the half step +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real V_calc (const amrex::Array4& U, int i, int j, int k, int comp, amrex::Real c) +{ + // comp -> x, y, z -> 0, 1, 2, return Vx, Vy, or Vz: + amrex::Real gamma = std::sqrt(1.0 + (U(i,j,k,1)*U(i,j,k,1) + U(i,j,k,2)*U(i,j,k,2) + U(i,j,k,3)*U(i,j,k,3))/(c*c)); + return U(i,j,k,comp+1)/gamma; +} +// mindmod +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real minmod (amrex::Real a, amrex::Real b) +{ + if (a > 0.0 && b > 0.0) + return std::min(a, b); + else if (a < 0.0 && b < 0.0) + return std::max(a, b); + else + return 0.0; +} +// Min of 3 inputs +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real min3 (amrex::Real a, amrex::Real b, amrex::Real c) +{ + return std::min(a, std::min(b, c) ); +} +// Max of 3 inputs +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real max3 (amrex::Real a, amrex::Real b, amrex::Real c) +{ + return std::max(a, std::max(b, c) ); +} +// mindmod +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real minmod3 (amrex::Real a, amrex::Real b , amrex::Real c) +{ + if (a > 0.0 && b > 0.0 && c > 0.0) + return min3(a,b,c); + else if (a < 0.0 && b < 0.0 && c < 0.0) + return max3(a,b,c); + else + return 0.0; +} +//maxmod +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real maxmod (amrex::Real a, amrex::Real b) +{ + if (a > 0.0 && b > 0.0) + return std::max(a, b); + else if (a < 0.0 && b < 0.0) + return std::min(a, b); + else + return 0.0; +} +// Rusanov Flux (density) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_N (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5*(Vm*Um(i,j,k,0) + Vp*Up(i,j,k,0)) - (0.5*c)*(Up(i,j,k,0) - Um(i,j,k,0)); +} +// Rusanov Flux (Momentum density x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_NUx (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5*(Vm*Um(i,j,k,0)*Um(i,j,k,1) + Vp*Up(i,j,k,0)*Up(i,j,k,1)) + - (0.5*c)*(Up(i,j,k,0)*Up(i,j,k,1) - Um(i,j,k,0)*Um(i,j,k,1)); +} +// Rusanov Flux (Momentum density y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_NUy (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5*(Vm*Um(i,j,k,0)*Um(i,j,k,2) + Vp*Up(i,j,k,0)*Up(i,j,k,2)) + - (0.5*c)*(Up(i,j,k,0)*Up(i,j,k,2) - Um(i,j,k,0)*Um(i,j,k,2)); +} +// Rusanov Flux (Momentum density z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_NUz (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5*(Vm*Um(i,j,k,0)*Um(i,j,k,3) + Vp*Up(i,j,k,0)*Up(i,j,k,3)) + - (0.5*c)*(Up(i,j,k,0)*Up(i,j,k,3) - Um(i,j,k,0)*Um(i,j,k,3)); +} +// ave_minmod high diffusivity, sigma can be between [1,2] +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave_adjustable_diff (amrex::Real a, amrex::Real b) +{ + amrex::Real sigma = 2.0*0.732050807568877; + if (a*b > 0.0) + return minmod3( (a+b)/2.0, sigma*a, sigma*b ); + else + return 0.0; +} +// ave_minmod Low diffusivity +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave (amrex::Real a, amrex::Real b) +{ + if (a*b > 0.0) + return minmod3( (a+b)/2.0, 2.0*a, 2.0*b ); + else + return 0.0; +} +// ave_superbee +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave_superbee (amrex::Real a, amrex::Real b) +{ + if (a*b > 0.0) + return minmod( maxmod(a,b), minmod(2.0*a,2.0*b)); + else + return 0.0; +} +// stage2 slope limiting +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave_stage2 (amrex::Real dQ, amrex::Real a, amrex::Real b, amrex::Real c) +{ + // sigma = sqrt(3) -1 + amrex::Real sigma = 0.732050807568877; + amrex::Real dq_min = 2.0*std::min( b - min3(a,b,c), max3(a,b,c) - b); + return ( std::abs(dQ)/dQ ) * std::min( std::abs(dQ) , sigma*std::abs(dq_min) ); +} +// Returns the offset indices for the "plus" grid +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void plus_index_offsets (int i, int j, int k, int& ip, int& jp, int& kp, int comp) +{ + // Find the correct offsets +#if defined(WARPX_DIM_3D) + if (comp == 0) { //x + ip = i - 1; jp = j; kp = k; + } else if (comp == 1){ //y + ip = i; jp = j - 1; kp = k; + } else if (comp == 2){ //z + ip = i; jp = j; kp = k - 1; + } + #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (comp == 0) { //x + ip = i - 1; jp = j; kp = k; + } else if (comp == 2){ //z + ip = i; jp = j - 1; kp = k; + } +#else + if (comp == 2) { //z + ip = i - 1; jp = j; kp = k; + } +#endif +} +// Compute the zero edges +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void compute_U_edges (const amrex::Array4& Um, const amrex::Array4& Up, int i, int j, int k, amrex::Box box, +amrex::Real U_tilde0, amrex::Real U_tilde1, amrex::Real U_tilde2, amrex::Real U_tilde3, +amrex::Real dU0x, amrex::Real dU1x, amrex::Real dU2x, amrex::Real dU3x, int comp) +{ + + // comp -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, comp); + + if ( box.contains(i,j,k) ) { + Um(i,j,k,0) = U_tilde0 + dU0x/2.0; + Um(i,j,k,1) = U_tilde1 + dU1x/2.0; + Um(i,j,k,2) = U_tilde2 + dU2x/2.0; + Um(i,j,k,3) = U_tilde3 + dU3x/2.0; + } + + if ( box.contains(ip,jp,kp) ) { + Up(ip,jp,kp,0) = U_tilde0 - dU0x/2.0; + Up(ip,jp,kp,1) = U_tilde1 - dU1x/2.0; + Up(ip,jp,kp,2) = U_tilde2 - dU2x/2.0; + Up(ip,jp,kp,3) = U_tilde3 - dU3x/2.0; + } +} +// Compute the zero edges +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void set_U_edges_to_zero (const amrex::Array4& Um, +const amrex::Array4& Up, int i, int j, int k, amrex::Box box, int comp) +{ + + // comp -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, comp); + + if ( box.contains(i,j,k) ) { + Um(i,j,k,0) = 0.0; + Um(i,j,k,1) = 0.0; + Um(i,j,k,2) = 0.0; + Um(i,j,k,3) = 0.0; + } + + if ( box.contains(ip,jp,kp) ) { + Up(ip,jp,kp,0) = 0.0; + Up(ip,jp,kp,1) = 0.0; + Up(ip,jp,kp,2) = 0.0; + Up(ip,jp,kp,3) = 0.0; + } +} +// Positivity Limiter +// if Q_minus or Q_plus is zero for the density (i.e. component 0 of Q_minus/Q_plus), set dQ to 0 and recompute Q_minus / Q_plus +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void positivity_limiter (const amrex::Array4& U_edge_plus, +const amrex::Array4& U_edge_minus, const amrex::Array4& N_arr, +int i, int j, int k, amrex::Box box, amrex::Real Ux, amrex::Real Uy, amrex::Real Uz, +int comp) +{ + + // comp -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, comp); + + // Set the edges to zero. If one edge in a cell is zero, we must self-consistently + // set the slope to zero (hence why we have the three cases, the first is when + // both points exist, and the second two are are edge cases) + if (( box.contains(i,j,k) ) && ( box.contains(ip,jp,kp) )) { + if ((U_edge_minus(i,j,k,0) < 0.0) || (U_edge_plus(ip,jp,kp,0) < 0.0)) { + U_edge_minus(i,j,k,0) = N_arr(i,j,k); + U_edge_minus(i,j,k,1) = Ux; + U_edge_minus(i,j,k,2) = Uy; + U_edge_minus(i,j,k,3) = Uz; + U_edge_plus(ip,jp,kp,0) = N_arr(i,j,k); + U_edge_plus(ip,jp,kp,1) = Ux; + U_edge_plus(ip,jp,kp,2) = Uy; + U_edge_plus(ip,jp,kp,3) = Uz; + } + } else if (( box.contains(i,j,k) ) && ( box.contains(ip,jp,kp) != 1)) { + if (U_edge_minus(i,j,k,0) < 0.0) { + U_edge_minus(i,j,k,0) = N_arr(i,j,k); + U_edge_minus(i,j,k,1) = Ux; + U_edge_minus(i,j,k,2) = Uy; + U_edge_minus(i,j,k,3) = Uz; + } + } else if (( box.contains(i,j,k) != 1 ) && ( box.contains(ip,jp,kp) )) { + if (U_edge_plus(ip,jp,kp,0) < 0.0){ + U_edge_plus(ip,jp,kp,0) = N_arr(i,j,k); + U_edge_plus(ip,jp,kp,1) = Ux; + U_edge_plus(ip,jp,kp,2) = Uy; + U_edge_plus(ip,jp,kp,3) = Uz; + } + } +} + +// Compute the difference in N (down-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDx_N (const amrex::Array4& N, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i,j,k) - N(i-1,j,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0; +#endif +} +// Compute the difference in N (up-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDx_N (const amrex::Array4& N, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i+1,j,k) - N(i,j,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0; +#endif +} +// Compute the difference in N (down-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDy_N (const amrex::Array4& N, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j,k) - N(i,j-1,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0; +#endif +} +// Compute the difference in N (up-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDy_N (const amrex::Array4& N, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j+1,k) - N(i,j,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0; +#endif +} +// Compute the difference in N (down-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDz_N (const amrex::Array4& N, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j,k) - N(i,j,k-1); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i,j,k) - N(i,j-1,k); +#else + return N(i,j,k) - N(i-1,j,k); +#endif +} +// Compute the difference in N (up-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDz_N (const amrex::Array4& N, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j,k+1) - N(i,j,k); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i,j+1,k) - N(i,j,k); +#else + return N(i+1,j,k) - N(i,j,k); +#endif +} + + +// Compute the difference in U (down-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDx_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_m = 0; + if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + return U - U_m; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0; +#endif +} +// Compute the difference in U (up-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDx_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_p = 0; + if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + return U_p - U; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0; +#endif +} + +// Compute the difference in U (down-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDy_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_m = 0; + if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + return U - U_m; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0; +#endif +} +// Compute the difference in U (up-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDy_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + // Write the correct differences +#if defined(WARPX_DIM_3D) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_p = 0; + if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + return U_p - U; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0; +#endif +} + +// Compute the difference in U (down-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDz_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + // Write the correct differences + amrex::Real U_m = 0; + + // U is zero if N is zero, Check positivity before dividing +#if defined(WARPX_DIM_3D) + if (N(i,j,k-1) > 0) U_m = NU(i,j,k-1)/N(i,j,k-1); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); +#else + if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); +#endif + + // Return the difference + return U - U_m; +} +// Compute the difference in U (up-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDz_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + // Write the correct differences + amrex::Real U_p = 0; + + // U is zero if N is zero, Check positivity before dividing +#if defined(WARPX_DIM_3D) + if (N(i,j,k+1) > 0) U_p = NU(i,j,k+1)/N(i,j,k+1); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); +#else + if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); +#endif + + // Return the difference + return U_p - U; +} + + +// Flux difference calculation +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real dF (const amrex::Array4& U_minus, +const amrex::Array4& U_plus,int i,int j,int k,amrex::Real clight, int comp, int dir) +{ + // dir -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, dir); + + amrex::Real V_L_minus = V_calc(U_minus,ip,jp,kp,dir,clight); + amrex::Real V_I_minus = V_calc(U_minus,i,j,k,dir,clight); + amrex::Real V_L_plus = V_calc(U_plus,ip,jp,kp,dir,clight); + amrex::Real V_I_plus = V_calc(U_plus,i,j,k,dir,clight); + + // Flux differences depending on the component to compute + if (comp == 0){ + return flux_N( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_N( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } else if (comp == 1){ + return flux_NUx( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_NUx( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } else if (comp == 2){ + return flux_NUy( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_NUy( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } else { //if (comp == 3) + return flux_NUz( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_NUz( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } +} + +#endif /*WARPX_MusclHancock_H_*/ diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H new file mode 100644 index 00000000000..28f37ae393b --- /dev/null +++ b/Source/Fluids/WarpXFluidContainer.H @@ -0,0 +1,186 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_WarpXFluidContainer_H_ +#define WARPX_WarpXFluidContainer_H_ + +#include "Evolve/WarpXDtType.H" +#include "Initialization/PlasmaInjector.H" +#include "MultiFluidContainer.H" + +#include +#include + +#include + + +/** + * WarpXFluidContainer is the base class from which all concrete + * fluid container classes derive. + * + * WarpXFluidContainer contains the main functions for initialization, + * interaction with the grid (field gather and current deposition), fluid + * source and push, advective update and updates for non-intertial terms. + */ +class WarpXFluidContainer +{ +public: + friend MultiFluidContainer; + + WarpXFluidContainer (int nlevs_max, int ispecies, const std::string& name); + ~WarpXFluidContainer() {} + + void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + + void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + + void ReadParameters (); + + /** + * Evolve updates a single timestep (dt) of the cold relativistic fluid eqautions + */ + void Evolve (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + amrex::Real cur_time, bool skip_deposition=false); + + /** + * AdvectivePush_Muscl takes a single timestep (dt) of the cold relativistic fluid equations + * using a Muscl-Handcock scheme + * + * \brief Advective term, cold-rel. fluids + * + * \param[in] lev refinement level + */ + void AdvectivePush_Muscl (int lev); + + + /** + * Apply (non-periodic) BC on the fluids (needed for spatial derivative), + * and communicate N, NU at boundaries + * + * \brief Apply non-periodic BC to fluids and communicate boundaries + * + * \param[in] lev refinement level + */ + void ApplyBcFluidsAndComms (int lev); + +#if defined(WARPX_DIM_RZ) + /** + * centrifugal_source_rz adds contributions due to curvature acceleration for a + * single timestep using an SSP-RK3 timestep for RZ specifically + * + * \brief Centrifugal source term + * + * \param[in] lev refinement level + */ + void centrifugal_source_rz (int lev); +#endif + + /** + * GatherAndPush introduces the Lorentz term in the cold relativistic fluid + * equations for a single timestep (dt) using Higuera and Cary Push + * + * \brief Lorentz Momentum Source + * + * \param[in] lev refinement level + * \param[in] Ex Yee electric field (x) + * \param[in] Ey Yee electric field (y) + * \param[in] Ez Yee electric field (z) + * \param[in] Bx Yee magnetic field (x) + * \param[in] By Yee magnetic field (y) + * \param[in] Bz Yee magnetic field (z) + * \param[in] cur_time Current time + */ + void GatherAndPush (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::Real cur_time); + + /** + * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and + * sums the contributions to the particle current density + * + * \brief Deposit fluid current density. + * + * \param[in] lev refinement level + * \param[in,out] jx current density MultiFab x comp. + * \param[in,out] jy current density MultiFab y comp. + * \param[in,out] jz current density MultiFab z comp. + */ + void DepositCurrent (int lev, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + + /** + * DepositCharge interpolates the fluid charge density onto the Yee grid and + * sums the contributions to the particle charge density + * + * \brief Deposit fluid charge density. + * + * \param[in] lev refinement level + * \param[in,out] rho charge density MultiFab. + */ + void DepositCharge (int lev, amrex::MultiFab &rho, int icomp = 0); + + amrex::Real getCharge () const {return charge;} + amrex::Real getMass () const {return mass;} + +protected: + int species_id; + std::string species_name; + amrex::Real charge; + amrex::Real mass; + + int do_not_push = 0; + int do_not_gather = 0; + int do_not_deposit = 0; + PhysicalSpecies physical_species; + + // Parser for external fields + std::string m_B_ext_s = "none"; + std::string m_E_ext_s = "none"; + + // Parser for B_external on the particle + std::unique_ptr m_Bx_parser; + std::unique_ptr m_By_parser; + std::unique_ptr m_Bz_parser; + amrex::ParserExecutor<4> m_Bxfield_parser; + amrex::ParserExecutor<4> m_Byfield_parser; + amrex::ParserExecutor<4> m_Bzfield_parser; + + // Parser for E_external on the particle + std::unique_ptr m_Ex_parser; + std::unique_ptr m_Ey_parser; + std::unique_ptr m_Ez_parser; + amrex::ParserExecutor<4> m_Exfield_parser; + amrex::ParserExecutor<4> m_Eyfield_parser; + amrex::ParserExecutor<4> m_Ezfield_parser; + + std::unique_ptr h_inj_rho; + InjectorDensity* d_inj_rho = nullptr; + std::unique_ptr density_parser; + + std::unique_ptr h_inj_mom; + InjectorMomentum* d_inj_mom = nullptr; + std::unique_ptr ux_parser; + std::unique_ptr uy_parser; + std::unique_ptr uz_parser; + + // Keep a pointer to TemperatureProperties to ensure the lifetime of the + // contained Parser + std::unique_ptr h_mom_temp; + std::unique_ptr h_mom_vel; + +public: + + // MultiFabs that contain the density (N) and momentum density (NU) of this fluid species, for each refinement level + amrex::Vector< std::unique_ptr > N; + amrex::Vector, 3 > > NU; + +}; + +#endif diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp new file mode 100644 index 00000000000..758e353208e --- /dev/null +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -0,0 +1,1353 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "ablastr/coarsen/sample.H" +#include "Particles/Pusher/UpdateMomentumHigueraCary.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include "MusclHancockUtils.H" +#include "Fluids/WarpXFluidContainer.H" +#include "WarpX.H" +#include +#include "Utils/Parser/ParserUtils.H" +#include "Utils/WarpXUtil.H" +#include "Utils/SpeciesUtils.H" + +using namespace ablastr::utils::communication; +using namespace amrex; + +WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std::string &name) +{ + species_id = ispecies; + species_name = name; + + ReadParameters(); + + // Initialize injection objects + const ParmParse pp_species_name(species_name); + SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, "none", h_inj_mom, + ux_parser, uy_parser, uz_parser, h_mom_temp, h_mom_vel); + if (h_inj_rho) { +#ifdef AMREX_USE_GPU + d_inj_rho = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorDensity))); + amrex::Gpu::htod_memcpy_async(d_inj_rho, h_inj_rho.get(), sizeof(InjectorDensity)); +#else + d_inj_rho = h_inj_rho.get(); +#endif + } + if (h_inj_mom) { +#ifdef AMREX_USE_GPU + d_inj_mom = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorMomentum))); + amrex::Gpu::htod_memcpy_async(d_inj_mom, h_inj_mom.get(), sizeof(InjectorMomentum)); +#else + d_inj_mom = h_inj_mom.get(); +#endif + } + amrex::Gpu::synchronize(); + + // Resize the list of MultiFabs for the right number of levels + N.resize(nlevs_max); + NU.resize(nlevs_max); +} + +void WarpXFluidContainer::ReadParameters() +{ + + // Extract charge, mass, species type + std::string injection_style = "none"; + SpeciesUtils::extractSpeciesProperties(species_name, injection_style, charge, mass, physical_species); + + const ParmParse pp_species_name(species_name); + pp_species_name.query("do_not_deposit", do_not_deposit); + pp_species_name.query("do_not_gather", do_not_gather); + pp_species_name.query("do_not_push", do_not_push); + + // default values of E_external and B_external + // are used to set the E and B field when "constant" or "parser" + // is not explicitly used in the input + pp_species_name.query("B_ext_init_style", m_B_ext_s); + std::transform(m_B_ext_s.begin(), + m_B_ext_s.end(), + m_B_ext_s.begin(), + ::tolower); + pp_species_name.query("E_ext_init_style", m_E_ext_s); + std::transform(m_E_ext_s.begin(), + m_E_ext_s.end(), + m_E_ext_s.begin(), + ::tolower); + + // Parse external fields: + // if the input string for B_ext_s is + // "parse_b_ext_function" then the mathematical expression + // for the Bx_, By_, Bz_external_function(x,y,z) + // must be provided in the input file. + if (m_B_ext_s == "parse_b_ext_function") { + // store the mathematical expression as string + std::string str_Bx_ext_function; + std::string str_By_ext_function; + std::string str_Bz_ext_function; + utils::parser::Store_parserString( + pp_species_name, "Bx_external_function(x,y,z,t)", + str_Bx_ext_function); + utils::parser::Store_parserString( + pp_species_name, "By_external_function(x,y,z,t)", + str_By_ext_function); + utils::parser::Store_parserString( + pp_species_name, "Bz_external_function(x,y,z,t)", + str_Bz_ext_function); + + // Parser for B_external on the fluid + m_Bx_parser = std::make_unique( + utils::parser::makeParser(str_Bx_ext_function,{"x","y","z","t"})); + m_By_parser = std::make_unique( + utils::parser::makeParser(str_By_ext_function,{"x","y","z","t"})); + m_Bz_parser = std::make_unique( + utils::parser::makeParser(str_Bz_ext_function,{"x","y","z","t"})); + + } + + // if the input string for E_ext_s is + // "parse_e_ext_function" then the mathematical expression + // for the Ex_, Ey_, Ez_external_function(x,y,z) + // must be provided in the input file. + if (m_E_ext_s == "parse_e_ext_function") { + // store the mathematical expression as string + std::string str_Ex_ext_function; + std::string str_Ey_ext_function; + std::string str_Ez_ext_function; + utils::parser::Store_parserString( + pp_species_name, "Ex_external_function(x,y,z,t)", + str_Ex_ext_function); + utils::parser::Store_parserString( + pp_species_name, "Ey_external_function(x,y,z,t)", + str_Ey_ext_function); + utils::parser::Store_parserString( + pp_species_name, "Ez_external_function(x,y,z,t)", + str_Ez_ext_function); + // Parser for E_external on the fluid + m_Ex_parser = std::make_unique( + utils::parser::makeParser(str_Ex_ext_function,{"x","y","z","t"})); + m_Ey_parser = std::make_unique( + utils::parser::makeParser(str_Ey_ext_function,{"x","y","z","t"})); + m_Ez_parser = std::make_unique( + utils::parser::makeParser(str_Ez_ext_function,{"x","y","z","t"})); + } +} + +void WarpXFluidContainer::AllocateLevelMFs(int lev, const BoxArray &ba, const DistributionMapping &dm) +{ + int ncomps = 1; + const amrex::IntVect nguards(AMREX_D_DECL(2, 2, 2)); + + // set human-readable tag for each MultiFab + auto const tag = [lev](std::string tagname) + { + tagname.append("[l=").append(std::to_string(lev)).append("]"); + return tagname; + }; + + WarpX::AllocInitMultiFab(N[lev], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid density"), 0.0_rt); + + WarpX::AllocInitMultiFab(NU[lev][0], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid momentum density [x]"), 0.0_rt); + WarpX::AllocInitMultiFab(NU[lev][1], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid momentum density [y]"), 0.0_rt); + WarpX::AllocInitMultiFab(NU[lev][2], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid momentum density [z]"), 0.0_rt); +} + +void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur_time) +{ + WARPX_PROFILE("WarpXFluidContainer::InitData"); + + // Convert initialization box to nodal box + init_box.surroundingNodes(); + + // Create local copies of pointers for GPU kernels + InjectorDensity* inj_rho = d_inj_rho; + InjectorMomentum* inj_mom = d_inj_mom; + + // Extract grid geometry properties + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); + const amrex::Real clight = PhysConst::c; + const amrex::Real gamma_boost = WarpX::gamma_boost; + const amrex::Real beta_boost = WarpX::beta_boost; + + // Loop through cells and initialize their value +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + // Return the intersection of all cells and the ones we wish to update + amrex::Box init_box_intersection = init_box & tile_box; + + amrex::ParallelFor(init_box_intersection, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + // Lorentz transform z (from boosted to lab frame) + if (gamma_boost > 1._rt){ + z = gamma_boost*(z + beta_boost*clight*cur_time); + } + + amrex::Real n = inj_rho->getDensity(x, y, z); + amrex::XDim3 u = inj_mom->getBulkMomentum(x, y, z); + + // Give u the right dimensions of m/s + u.x = u.x * clight; + u.y = u.y * clight; + u.z = u.z * clight; + + // Check if n > 0 and if not, don't compute the boost + // Lorentz transform n, u (from lab to boosted frame) + if (n > 0.0){ + if (gamma_boost > 1._rt){ + amrex::Real gamma = sqrt(1.0 + (u.x*u.x + u.y*u.y + u.z*u.z)/(clight*clight)); + amrex::Real n_boosted = gamma_boost*n*( 1.0 - beta_boost*u.z/(gamma*clight) ); + amrex::Real uz_boosted = gamma_boost*(u.z - beta_boost*clight*gamma); + u.z = uz_boosted; + n = n_boosted; + } + } + + // Multiply by clight so u is back in SI units + N_arr(i, j, k) = n; + NUx_arr(i, j, k) = n * u.x; + NUy_arr(i, j, k) = n * u.y; + NUz_arr(i, j, k) = n * u.z; + + } + ); + } +} + + +void WarpXFluidContainer::Evolve( + int lev, + const amrex::MultiFab &Ex, const amrex::MultiFab &Ey, const amrex::MultiFab &Ez, + const amrex::MultiFab &Bx, const amrex::MultiFab &By, const amrex::MultiFab &Bz, + amrex::MultiFab* rho, amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, + amrex::Real cur_time, bool skip_deposition) +{ + + WARPX_PROFILE("WarpXFluidContainer::Evolve"); + + if (rho && ! skip_deposition && ! do_not_deposit) { + // Deposit charge before particle push, in component 0 of MultiFab rho. + DepositCharge(lev, *rho, 0); + } + + // Step the Lorentz Term + if(!do_not_gather){ + GatherAndPush(lev, Ex, Ey, Ez, Bx, By, Bz, cur_time); + } + + // Cylindrical centrifugal term + if(!do_not_push){ +#if defined(WARPX_DIM_RZ) + centrifugal_source_rz(lev); +#endif + + // Apply (non-periodic) BC on the fluids (needed for spatial derivative), + // and communicate N, NU at boundaries + ApplyBcFluidsAndComms(lev); + + // Step the Advective term + AdvectivePush_Muscl(lev); + } + + // Deposit rho to the simulation mesh + // Deposit charge (end of the step) + if (rho && ! skip_deposition && ! do_not_deposit) { + DepositCharge(lev, *rho, 1); + } + + // Deposit J to the simulation mesh + if (!skip_deposition && ! do_not_deposit) { + DepositCurrent(lev, jx, jy, jz); + } +} + +// Momentum source due to curvature +void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) +{ + WARPX_PROFILE("WarpXFluidContainer::ApplyBcFluidsAndComms"); + + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const amrex::Periodicity &period = geom.periodicity(); + const Array periodic_directions = geom.isPeriodic(); + amrex::Box domain = geom.Domain(); + // Convert to nodal box + domain.surroundingNodes(); + + // H&C push the momentum +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + + //Grow the tilebox + tile_box.grow(1); + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // If the cell is is first gaurd cell & the dimension is non + // periodic, then copy Q_{i+1} = Q_{i-1}. + // Don't check r-dir in Z: +#if defined(WARPX_DIM_3D) + + // Upper end (index 2) + if ( (periodic_directions[2] != 1) && (k==domain.bigEnd(2)+1) ){ + N_arr(i,j,k) = N_arr(i,j,k-2); + NUx_arr(i,j,k) = NUx_arr(i,j,k-2); + NUy_arr(i,j,k) = NUy_arr(i,j,k-2); + NUz_arr(i,j,k) = NUz_arr(i,j,k-2); + + // Lower end (index 2) + } else if ( (periodic_directions[2] != 1) && (k==domain.smallEnd(2)-1) ) { + N_arr(i,j,k) = N_arr(i,j,k+2); + NUx_arr(i,j,k) = NUx_arr(i,j,k+2); + NUy_arr(i,j,k) = NUy_arr(i,j,k+2); + NUz_arr(i,j,k) = NUz_arr(i,j,k+2); + } + +#elif ( defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) ) + + // Upper end (index 1) + if ( (periodic_directions[1] != 1) && (j==domain.bigEnd(1)+1) ){ + N_arr(i,j,k) = N_arr(i,j-2,k); + NUx_arr(i,j,k) = NUx_arr(i,j-2,k); + NUy_arr(i,j,k) = NUy_arr(i,j-2,k); + NUz_arr(i,j,k) = NUz_arr(i,j-2,k); + + // Lower end (index 1`) + } else if ( (periodic_directions[1] != 1) && (j==domain.smallEnd(1)-1) ) { + N_arr(i,j,k) = N_arr(i,j+2,k); + NUx_arr(i,j,k) = NUx_arr(i,j+2,k); + NUy_arr(i,j,k) = NUy_arr(i,j+2,k); + NUz_arr(i,j,k) = NUz_arr(i,j+2,k); + + } + +#elif ( defined(WARPX_DIM_1D_Z) || defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) ) + + // Upper end (index 0) + if ( (periodic_directions[0] != 1) && (i==domain.bigEnd(0)+1) ){ + N_arr(i,j,k) = N_arr(i-2,j,k); + NUx_arr(i,j,k) = NUx_arr(i-2,j,k); + NUy_arr(i,j,k) = NUy_arr(i-2,j,k); + NUz_arr(i,j,k) = NUz_arr(i-2,j,k); + + // Lower end (index 0) + } else if ( (periodic_directions[0] != 1) && (i==domain.smallEnd(0)-1) ) { + N_arr(i,j,k) = N_arr(i+2,j,k); + NUx_arr(i,j,k) = NUx_arr(i+2,j,k); + NUy_arr(i,j,k) = NUy_arr(i+2,j,k); + NUz_arr(i,j,k) = NUz_arr(i+2,j,k); + } + +#else + +#endif + } + ); + } + + // Fill guard cells + FillBoundary(*N[lev], N[lev]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*NU[lev][0], NU[lev][0]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*NU[lev][1], NU[lev][1]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*NU[lev][2], NU[lev][2]->nGrowVect(), WarpX::do_single_precision_comms, period); +} + +// Muscl Advection Update +void WarpXFluidContainer::AdvectivePush_Muscl (int lev) +{ + WARPX_PROFILE("WarpXFluidContainer::AdvectivePush_Muscl"); + + // Grab the grid spacing + WarpX &warpx = WarpX::GetInstance(); + const Real dt = warpx.getdt(lev); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const amrex::Real clight = PhysConst::c; +#if defined(WARPX_DIM_3D) + amrex::Real dt_over_dx = (dt/dx[0]); + amrex::Real dt_over_dy = (dt/dx[1]); + amrex::Real dt_over_dz = (dt/dx[2]); + amrex::Real dt_over_dx_half = 0.5*(dt/dx[0]); + amrex::Real dt_over_dy_half = 0.5*(dt/dx[1]); + amrex::Real dt_over_dz_half = 0.5*(dt/dx[2]); +#elif defined(WARPX_DIM_XZ) + amrex::Real dt_over_dx_half = 0.5*(dt/dx[0]); + amrex::Real dt_over_dz_half = 0.5*(dt/dx[1]); + amrex::Real dt_over_dx = (dt/dx[0]); + amrex::Real dt_over_dz = (dt/dx[1]); +#elif defined(WARPX_DIM_RZ) + const auto problo = geom.ProbLoArray(); + amrex::Real dt_over_dx_half = 0.5*(dt/dx[0]); + amrex::Real dt_over_dz_half = 0.5*(dt/dx[1]); + amrex::Box const& domain = geom.Domain(); +#else + amrex::Real dt_over_dz = (dt/dx[0]); + amrex::Real dt_over_dz_half = 0.5*(dt/dx[0]); +#endif + + amrex::BoxArray ba = N[lev]->boxArray(); + + // Temporary Half-step values +#if defined(WARPX_DIM_3D) + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); +#else + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); +#endif + + // Fill edge values of N and U at the half timestep for MUSCL +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + // Loop over a box with one extra gridpoint in the ghost region to avoid + // an extra MPI communication between the edge value computation loop and + // the flux calculation loop + amrex::Box tile_box = mfi.growntilebox(1); + + // Limit the grown box for RZ at r = 0, r_max +#if defined (WARPX_DIM_RZ) + const int idir = 0; + const int n_cell = -1; + tile_box.growLo(idir, n_cell); + tile_box.growHi(idir, n_cell); +#endif + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + // Boxes are computed to avoid going out of bounds. + // Grow the entire domain + amrex::Box box = mfi.validbox(); + box.grow(1); +#if defined(WARPX_DIM_3D) + amrex::Box const box_x = amrex::convert( box, tmp_U_minus_x.ixType() ); + amrex::Box const box_y = amrex::convert( box, tmp_U_minus_y.ixType() ); + amrex::Box const box_z = amrex::convert( box, tmp_U_minus_z.ixType() ); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Box const box_x = amrex::convert( box, tmp_U_minus_x.ixType() ); + amrex::Box const box_z = amrex::convert( box, tmp_U_minus_z.ixType() ); +#else + amrex::Box const box_z = amrex::convert( box, tmp_U_minus_z.ixType() ); +#endif + + //N and NU are always defined at the nodes, the tmp_Q_* are defined + //inbetween the nodes (i.e. on the staggered Yee grid) and store the + //values of N and U at these points. + //(i.e. the 4 components correspond to N + the 3 components of U) + // Extract the temporary arrays for edge values +#if defined(WARPX_DIM_3D) + amrex::Array4 U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 U_minus_y = tmp_U_minus_y.array(mfi); + amrex::Array4 U_plus_y = tmp_U_plus_y.array(mfi); + amrex::Array4 U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 U_plus_z = tmp_U_plus_z.array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array4 U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 U_plus_z = tmp_U_plus_z.array(mfi); +#else + amrex::Array4 U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 U_plus_z = tmp_U_plus_z.array(mfi); +#endif + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Density positivity check (Makes the algorithm safe from divide by zeros) + if( N_arr(i,j,k) > 0.0){ + + // - Grab local Uz Uy Ux gamma + // Isolate U from NU + amrex::Real Ux = (NUx_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real Uy = (NUy_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real Uz = (NUz_arr(i, j, k) / N_arr(i,j,k)); + + // Compute useful quantities for J + amrex::Real c_sq = clight*clight; + amrex::Real gamma = sqrt(1.0 + (Ux*Ux + Uy*Uy + Uz*Uz)/(c_sq) ); + amrex::Real inv_c2_gamma3 = 1./(c_sq*gamma*gamma*gamma); + + // J represents are 4x4 matrices that show up in the advection + // equations written as a function of U = {N, Ux, Uy, Uz}: + // \partial_t U + Jx \partial_x U + Jy \partial_y U + Jz \partial_z U = 0 +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + amrex::Real Vx = Ux/gamma; + // Compute the non-zero element of Jx + amrex::Real J00x = Vx; + amrex::Real J01x = N_arr(i,j,k)*(1/gamma)*(1-Vx*Vx/c_sq); + amrex::Real J02x = -N_arr(i,j,k)*Uy*Ux*inv_c2_gamma3; + amrex::Real J03x = -N_arr(i,j,k)*Uz*Ux*inv_c2_gamma3; + amrex::Real J11x = Vx; + amrex::Real J22x = Vx; + amrex::Real J33x = Vx; + + // Compute the cell slopes x + amrex::Real dU0x = ave( DownDx_N(N_arr,i,j,k), UpDx_N(N_arr,i,j,k) ); + amrex::Real dU1x = ave( DownDx_U(N_arr,NUx_arr,Ux,i,j,k), UpDx_U(N_arr,NUx_arr,Ux,i,j,k) ); + amrex::Real dU2x = ave( DownDx_U(N_arr,NUy_arr,Uy,i,j,k), UpDx_U(N_arr,NUy_arr,Uy,i,j,k) ); + amrex::Real dU3x = ave( DownDx_U(N_arr,NUz_arr,Uz,i,j,k), UpDx_U(N_arr,NUz_arr,Uz,i,j,k) ); + +#endif + +#if defined(WARPX_DIM_3D) + amrex::Real Vy = Uy/gamma; + // Compute the non-zero element of Jy + amrex::Real J00y = Vy; + amrex::Real J01y = -N_arr(i,j,k)*Ux*Uy*inv_c2_gamma3; + amrex::Real J02y = N_arr(i,j,k)*(1/gamma)*(1-Vy*Vy/c_sq); + amrex::Real J03y = -N_arr(i,j,k)*Uz*Uy*inv_c2_gamma3; + amrex::Real J11y = Vy; + amrex::Real J22y = Vy; + amrex::Real J33y = Vy; + + // Compute the cell slopes y + amrex::Real dU0y = ave( DownDy_N(N_arr,i,j,k), UpDy_N(N_arr,i,j,k) ); + amrex::Real dU1y = ave( DownDy_U(N_arr,NUx_arr,Ux,i,j,k), UpDy_U(N_arr,NUx_arr,Ux,i,j,k) ); + amrex::Real dU2y = ave( DownDy_U(N_arr,NUy_arr,Uy,i,j,k), UpDy_U(N_arr,NUy_arr,Uy,i,j,k) ); + amrex::Real dU3y = ave( DownDy_U(N_arr,NUz_arr,Uz,i,j,k), UpDy_U(N_arr,NUz_arr,Uz,i,j,k) ); + +#endif + amrex::Real Vz = Uz/gamma; + // Compute the non-zero element of Jz + amrex::Real J00z = Vz; + amrex::Real J01z = -N_arr(i,j,k)*Ux*Uz*inv_c2_gamma3; + amrex::Real J02z = -N_arr(i,j,k)*Uy*Uz*inv_c2_gamma3; + amrex::Real J03z = N_arr(i,j,k)*(1/gamma)*(1-Vz*Vz/c_sq); + amrex::Real J11z = Vz; + amrex::Real J22z = Vz; + amrex::Real J33z = Vz; + + // Compute the cell slopes z + amrex::Real dU0z = ave( DownDz_N(N_arr,i,j,k), UpDz_N(N_arr,i,j,k) ); + amrex::Real dU1z = ave( DownDz_U(N_arr,NUx_arr,Ux,i,j,k), UpDz_U(N_arr,NUx_arr,Ux,i,j,k) ); + amrex::Real dU2z = ave( DownDz_U(N_arr,NUy_arr,Uy,i,j,k), UpDz_U(N_arr,NUy_arr,Uy,i,j,k) ); + amrex::Real dU3z = ave( DownDz_U(N_arr,NUz_arr,Uz,i,j,k), UpDz_U(N_arr,NUz_arr,Uz,i,j,k) ); + + + // Select the specific implementation depending on dimensionality +#if defined(WARPX_DIM_3D) + + // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; + amrex::Real JdU1x = J11x*dU1x ; + amrex::Real JdU2x = J22x*dU2x ; + amrex::Real JdU3x = J33x*dU3x; + amrex::Real JdU0y = J00y*dU0y + J01y*dU1y + J02y*dU2y + J03y*dU3y; + amrex::Real JdU1y = J11y*dU1y; + amrex::Real JdU2y = J22y*dU2y; + amrex::Real JdU3y = J33y*dU3y; + amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; + amrex::Real JdU1z = J11z*dU1z; + amrex::Real JdU2z = J22z*dU2z; + amrex::Real JdU3z = J33z*dU3z; + amrex::Real U_tilde0 = N_arr(i,j,k) - dt_over_dx_half*JdU0x - dt_over_dy_half*JdU0y - dt_over_dz_half*JdU0z; + amrex::Real U_tilde1 = Ux - dt_over_dx_half*JdU1x - dt_over_dy_half*JdU1y - dt_over_dz_half*JdU1z; + amrex::Real U_tilde2 = Uy - dt_over_dx_half*JdU2x - dt_over_dy_half*JdU2y - dt_over_dz_half*JdU2z; + amrex::Real U_tilde3 = Uz - dt_over_dx_half*JdU3x - dt_over_dy_half*JdU3y - dt_over_dz_half*JdU3z; + + + // Predict U at the cell edges (x) + compute_U_edges(U_minus_x, U_plus_x, i, j, k, box_x, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0x, dU1x, dU2x, dU3x,0); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_x, U_minus_x, N_arr, i, j, k, box_x, Ux, Uy, Uz, 0); + + // Predict U at the cell edges (y) + compute_U_edges(U_minus_y, U_plus_y, i, j, k, box_y, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0y, dU1y, dU2y, dU3y,1); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_y, U_minus_y, N_arr, i, j, k, box_y, Ux, Uy, Uz, 1); + + // Predict U at the cell edges (z) + compute_U_edges(U_minus_z, U_plus_z, i, j, k, box_z, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0z, dU1z, dU2z, dU3z,2); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_z, U_minus_z, N_arr, i, j, k, box_z, Ux, Uy, Uz, 2); + +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + + // Have no RZ-intertial source for primative vars if in XZ + amrex::Real N_source = 0.0; + +#if defined(WARPX_DIM_RZ) + amrex::Real dr = dx[0]; + amrex::Real r = problo[0] + i * dr; + // Impose "none" boundaries + // Condition: dUx = 0 at r = 0 + if (i == domain.smallEnd(0)) { + // R|_{0+} -> L|_{0-} + // N -> N (N_arr(i-1,j,k) -> N_arr(i+1,j,k)) + // NUr -> -NUr (NUx_arr(i-1,j,k) -> -NUx_arr(i+1,j,k)) + // NUt -> -NUt (NUy_arr(i-1,j,k) -> -NUy_arr(i+1,j,k)) + // NUz -> -NUz (NUz_arr(i-1,j,k) -> NUz_arr(i+1,j,k)) + dU0x = ave( -UpDx_N(N_arr,i,j,k) , UpDx_N(N_arr,i,j,k) ); + // First term in the ave is: U_{x,y} + U_{x,y}_p, + // which can be writen as 2*U_{x,y} + UpDx_U(U_{x,y}) + dU1x = ave( 2.0*Ux + UpDx_U(N_arr,NUx_arr,Ux,i,j,k) , UpDx_U(N_arr,NUx_arr,Ux,i,j,k) ); + dU2x = ave( 2.0*Uy + UpDx_U(N_arr,NUy_arr,Uy,i,j,k) , UpDx_U(N_arr,NUy_arr,Uy,i,j,k) ); + dU3x = ave( -UpDx_U(N_arr,NUz_arr,Uz,i,j,k) , UpDx_U(N_arr,NUz_arr,Uz,i,j,k) ); + } else if (i == domain.bigEnd(0)+1) { + dU0x = ave( DownDx_N(N_arr,i,j,k) , 0.0 ); + dU1x = ave( DownDx_U(N_arr,NUx_arr,Ux,i,j,k) , 0.0 ); + dU2x = ave( DownDx_U(N_arr,NUy_arr,Uy,i,j,k) , 0.0 ); + dU3x = ave( DownDx_U(N_arr,NUz_arr,Uz,i,j,k) , 0.0 ); + } + + // RZ sources: + if (i != domain.smallEnd(0)) { + N_source = N_arr(i,j,k)*Vx/r; + } +#endif + + // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; + amrex::Real JdU1x = J11x*dU1x; + amrex::Real JdU2x = J22x*dU2x; + amrex::Real JdU3x = J33x*dU3x; + amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; + amrex::Real JdU1z = J11z*dU1z; + amrex::Real JdU2z = J22z*dU2z; + amrex::Real JdU3z = J33z*dU3z; + amrex::Real U_tilde0 = N_arr(i,j,k) - dt_over_dx_half*JdU0x - dt_over_dz_half*JdU0z - (dt/2.0)*N_source; + amrex::Real U_tilde1 = Ux - dt_over_dx_half*JdU1x - dt_over_dz_half*JdU1z; + amrex::Real U_tilde2 = Uy - dt_over_dx_half*JdU2x - dt_over_dz_half*JdU2z; + amrex::Real U_tilde3 = Uz - dt_over_dx_half*JdU3x - dt_over_dz_half*JdU3z; + + // Predict U at the cell edges (x) + compute_U_edges(U_minus_x, U_plus_x, i, j, k, box_x, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0x, dU1x, dU2x, dU3x,0); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_x, U_minus_x, N_arr, i, j, k, box_x, Ux, Uy, Uz, 0); + + // Predict U at the cell edges (z) + compute_U_edges(U_minus_z, U_plus_z, i, j, k, box_z, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0z, dU1z, dU2z, dU3z,2); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_z, U_minus_z, N_arr, i, j, k, box_z, Ux, Uy, Uz, 2); + +#else + + // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; + amrex::Real JdU1z = J11z*dU1z; + amrex::Real JdU2z = J22z*dU2z; + amrex::Real JdU3z = J33z*dU3z; + amrex::Real U_tilde0 = N_arr(i,j,k) - dt_over_dz_half*JdU0z; + amrex::Real U_tilde1 = Ux - dt_over_dz_half*JdU1z; + amrex::Real U_tilde2 = Uy - dt_over_dz_half*JdU2z; + amrex::Real U_tilde3 = Uz - dt_over_dz_half*JdU3z; + + // Predict U at the cell edges (z) + compute_U_edges(U_minus_z, U_plus_z, i, j, k, box_z, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0z, dU1z, dU2z, dU3z,2); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_z, U_minus_z, N_arr, i, j, k, box_z, Ux, Uy, Uz, 2); + +#endif + // If N<= 0 then set the edge values (U_minus/U_plus) to zero + } else { +#if defined(WARPX_DIM_3D) + set_U_edges_to_zero(U_minus_x, U_plus_x, i, j, k, box_x, 0); + set_U_edges_to_zero(U_minus_y, U_plus_y, i, j, k, box_y, 1); + set_U_edges_to_zero(U_minus_z, U_plus_z, i, j, k, box_z, 2); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + set_U_edges_to_zero(U_minus_x, U_plus_x, i, j, k, box_x, 0); + set_U_edges_to_zero(U_minus_z, U_plus_z, i, j, k, box_z, 2); +#else + set_U_edges_to_zero(U_minus_z, U_plus_z, i, j, k, box_z, 2); +#endif + } + } + ); + } + + // Given the values of `U_minus` and `U_plus`, compute fluxes inbetween nodes, and update N, NU accordingly +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Array4 N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + +#if defined(WARPX_DIM_3D) + amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 const &U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 const &U_minus_y = tmp_U_minus_y.array(mfi); + amrex::Array4 const &U_plus_y = tmp_U_plus_y.array(mfi); + amrex::Array4 const &U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 const &U_plus_z = tmp_U_plus_z.array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 const &U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 const &U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 const &U_plus_z = tmp_U_plus_z.array(mfi); +#else + amrex::Array4 const &U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 const &U_plus_z = tmp_U_plus_z.array(mfi); +#endif + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Select the specific implementation depending on dimensionality +#if defined(WARPX_DIM_3D) + + // Update the conserved variables Q = [N, NU] from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,0,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,0,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,0,2); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,1,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,1,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,1,2); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,2,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,2,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,2,2); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,3,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,3,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,3,2); + +#elif defined(WARPX_DIM_XZ) + + // Update the conserved variables Q = [N, NU] from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,0,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,0,2); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,1,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,1,2); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,2,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,2,2); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,3,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,3,2); + +#elif defined(WARPX_DIM_RZ) + + // Compute the flux areas for RZ + // Cell-centered radius + amrex::Real dr = dx[0]; + amrex::Real dz = dx[1]; + amrex::Real r = problo[0] + i * dr; + amrex::Real Vij = 0.0; + amrex::Real S_Az = 0.0; + + // Volume element and z-facing surfaces + if (i == domain.smallEnd(0)) { + Vij = 2.0*MathConst::pi*(dr/2.0)*(dr/4.0)*dz; + S_Az = 2.0*MathConst::pi*(dr/4.0)*(dr/2.0); + } else if (i == domain.bigEnd(0)+1) { + Vij = 2.0*MathConst::pi*(r - dr/4.0)*(dr/2.0)*dz; + S_Az = 2.0*MathConst::pi*(r - dr/4.0)*(dr/2.0); + } else { + Vij = 2.0*MathConst::pi*r*dr*dz; + S_Az = 2.0*MathConst::pi*(r)*dr; + } + + // Radial Surfaces + amrex::Real S_Ar_plus = 2.0*MathConst::pi*(r + dr/2.0)*dz; + amrex::Real S_Ar_minus = 2.0*MathConst::pi*(r - dr/2.0)*dz; + if (i == domain.smallEnd(0)) + S_Ar_minus = 0.0; + if (i == domain.bigEnd(0)+1) + S_Ar_plus = 2.0*MathConst::pi*(r)*dz; + + // Impose "none" boundaries + // Condition: Vx(r) = 0 at boundaries + amrex::Real Vx_I_minus = V_calc(U_minus_x,i,j,k,0,clight); + amrex::Real Vx_L_plus = V_calc(U_plus_x,i-1,j,k,0,clight); + + // compute the fluxes: + // (note that _plus is shifted due to grid location) + amrex::Real Vx_L_minus = 0.0, Vx_I_plus = 0.0; + amrex::Real F0_minusx = 0.0, F1_minusx = 0.0, F2_minusx = 0.0, F3_minusx = 0.0; + amrex::Real F0_plusx = 0.0, F1_plusx = 0.0, F2_plusx = 0.0, F3_plusx = 0.0; + if (i != domain.smallEnd(0)) { + Vx_L_minus = V_calc(U_minus_x,i-1,j,k,0,clight); + F0_minusx = flux_N( U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + F1_minusx = flux_NUx(U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + F2_minusx = flux_NUy(U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + F3_minusx = flux_NUz(U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + } + if (i < domain.bigEnd(0)) { + Vx_I_plus = V_calc(U_plus_x,i,j,k,0,clight); + F0_plusx = flux_N( U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + F1_plusx = flux_NUx(U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + F2_plusx = flux_NUy(U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + F3_plusx = flux_NUz(U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + } + + // Update the conserved variables from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - (dt/Vij)*(F0_plusx - F0_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,0,2)*S_Az); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - (dt/Vij)*(F1_plusx - F1_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,1,2)*S_Az); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - (dt/Vij)*(F2_plusx - F2_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,2,2)*S_Az); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - (dt/Vij)*(F3_plusx - F3_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,3,2)*S_Az); + +#else + + // Update the conserved variables Q = [N, NU] from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,0,2); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,1,2); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,2,2); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,3,2); +#endif + } + ); + } +} + + +// Momentum source due to curvature +#if defined(WARPX_DIM_RZ) +void WarpXFluidContainer::centrifugal_source_rz (int lev) +{ + WARPX_PROFILE("WarpXFluidContainer::centrifugal_source_rz"); + + WarpX &warpx = WarpX::GetInstance(); + const Real dt = warpx.getdt(lev); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); + const amrex::Real clight = PhysConst::c; + amrex::Box const& domain = geom.Domain(); + + // H&C push the momentum +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Verify density is non-zero + if (N_arr(i,j,k)>0.0) { + + // Compute r + amrex::Real r = problo[0] + i * dx[0]; + + // Isolate U from NU + amrex::Real u_r = (NUx_arr(i, j, k) / (N_arr(i,j,k) * clight )); + amrex::Real u_theta = (NUy_arr(i, j, k) / (N_arr(i,j,k) * clight )); + amrex::Real u_z = (NUz_arr(i, j, k) / (N_arr(i,j,k) * clight )); + + // (SSP-RK3) Push the fluid momentum (R and Theta) + // F_r, F_theta are first order euler pushes of our rhs operator + if (i != domain.smallEnd(0)) { + amrex::Real u_r_1 = F_r(r,u_r,u_theta,u_z,dt); + amrex::Real u_theta_1 = F_theta(r,u_r,u_theta,u_z,dt); + amrex::Real u_r_2 = (0.75)*(u_r) + (0.25)*F_r(r,u_r_1,u_theta_1,u_z,dt); + amrex::Real u_theta_2 = (0.75)*(u_theta) + (0.25)*F_theta(r,u_r_1,u_theta_1,u_z,dt); + u_r = (1.0/3.0)*(u_r) + (2.0/3.0)*F_r(r,u_r_2,u_theta_2,u_z,dt); + u_theta = (1.0/3.0)*(u_theta) + (2.0/3.0)*F_theta(r,u_r_2,u_theta_2,u_z,dt); + + // Calculate NU, save NUr, NUtheta + NUx_arr(i,j,k) = N_arr(i,j,k)*u_r*clight; + NUy_arr(i,j,k) = N_arr(i,j,k)*u_theta*clight; + + // BC r = 0, u_theta = 0, and there is no extra source terms + } else { + NUx_arr(i,j,k) = 0.0; + NUy_arr(i,j,k) = 0.0; + } + } + } + ); + } +} +#endif + +// Momentum source from fields +void WarpXFluidContainer::GatherAndPush ( + int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + Real t) +{ + WARPX_PROFILE("WarpXFluidContainer::GatherAndPush"); + + WarpX &warpx = WarpX::GetInstance(); + const amrex::Real q = getCharge(); + const amrex::Real m = getMass(); + const Real dt = warpx.getdt(lev); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); + const amrex::Real gamma_boost = WarpX::gamma_boost; + const amrex::Real beta_boost = WarpX::beta_boost; + //Check whether m_E_ext_s is "none" + bool external_e_fields; // Needs intializing + bool external_b_fields; // Needs intializing + + + // Prepare interpolation of current components to cell center + amrex::GpuArray Nodal_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray Ex_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray Ey_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray Ez_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray Bx_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray By_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray Bz_type = amrex::GpuArray{0, 0, 0}; + for (int i = 0; i < AMREX_SPACEDIM; ++i) + { + Nodal_type[i] = N[lev]->ixType()[i]; + Ex_type[i] = Ex.ixType()[i]; + Ey_type[i] = Ey.ixType()[i]; + Ez_type[i] = Ez.ixType()[i]; + Bx_type[i] = Bx.ixType()[i]; + By_type[i] = By.ixType()[i]; + Bz_type[i] = Bz.ixType()[i]; + } + + // External field parsers + external_e_fields = (m_E_ext_s == "parse_e_ext_function"); + external_b_fields = (m_B_ext_s == "parse_b_ext_function"); + amrex::ParserExecutor<4> Exfield_parser; + amrex::ParserExecutor<4> Eyfield_parser; + amrex::ParserExecutor<4> Ezfield_parser; + amrex::ParserExecutor<4> Bxfield_parser; + amrex::ParserExecutor<4> Byfield_parser; + amrex::ParserExecutor<4> Bzfield_parser; + if (external_e_fields){ + constexpr int num_arguments = 4; //x,y,z,t + Exfield_parser = m_Ex_parser->compile(); + Eyfield_parser = m_Ey_parser->compile(); + Ezfield_parser = m_Ez_parser->compile(); + } + + if (external_b_fields){ + constexpr int num_arguments = 4; //x,y,z,t + Bxfield_parser = m_Bx_parser->compile(); + Byfield_parser = m_By_parser->compile(); + Bzfield_parser = m_Bz_parser->compile(); + } + + + // H&C push the momentum +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + + amrex::Array4 const& Ex_arr = Ex.array(mfi); + amrex::Array4 const& Ey_arr = Ey.array(mfi); + amrex::Array4 const& Ez_arr = Ez.array(mfi); + amrex::Array4 const& Bx_arr = Bx.array(mfi); + amrex::Array4 const& By_arr = By.array(mfi); + amrex::Array4 const& Bz_arr = Bz.array(mfi); + + // Here, we do not perform any coarsening. + amrex::GpuArray coarsening_ratio = {1, 1, 1}; + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Only run if density is positive + if (N_arr(i,j,k)>0.0) { + + // Interpolate fields from tmp to Nodal points + amrex::Real Ex_Nodal = ablastr::coarsen::sample::Interp(Ex_arr, + Ex_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Ey_Nodal = ablastr::coarsen::sample::Interp(Ey_arr, + Ey_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Ez_Nodal = ablastr::coarsen::sample::Interp(Ez_arr, + Ez_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Bx_Nodal = ablastr::coarsen::sample::Interp(Bx_arr, + Bx_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real By_Nodal = ablastr::coarsen::sample::Interp(By_arr, + By_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Bz_Nodal = ablastr::coarsen::sample::Interp(Bz_arr, + Bz_type, Nodal_type, coarsening_ratio, i, j, k, 0); + + if (gamma_boost > 1._rt) { // Lorentz transform fields due to moving frame + if ( ( external_b_fields ) || ( external_e_fields ) ){ + + // Lorentz transform z (from boosted to lab frame) + amrex::Real Ex_ext_boost, Ey_ext_boost, Ez_ext_boost; + amrex::Real Bx_ext_boost, By_ext_boost, Bz_ext_boost; + amrex::Real Ex_ext_lab, Ey_ext_lab, Ez_ext_lab; + amrex::Real Bx_ext_lab, By_ext_lab, Bz_ext_lab; + + // Grab the location +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + // Get the lab frame E and B + // Transform (boosted to lab) + amrex::Real t_lab = gamma_boost*(t + beta_boost*z/PhysConst::c); + amrex::Real z_lab = gamma_boost*(z + beta_boost*PhysConst::c*t); + + // Grab the external fields in the lab frame: + if ( external_e_fields ) { + Ex_ext_lab = Exfield_parser(x, y, z_lab, t_lab); + Ey_ext_lab = Eyfield_parser(x, y, z_lab, t_lab); + Ez_ext_lab = Ezfield_parser(x, y, z_lab, t_lab); + }else{ + Ex_ext_lab = 0.0; + Ey_ext_lab = 0.0; + Ez_ext_lab = 0.0; + } + if ( external_b_fields ) { + Bx_ext_lab = Bxfield_parser(x, y, z_lab, t_lab); + By_ext_lab = Byfield_parser(x, y, z_lab, t_lab); + Bz_ext_lab = Bzfield_parser(x, y, z_lab, t_lab); + }else{ + Bx_ext_lab = 0.0; + By_ext_lab = 0.0; + Bz_ext_lab = 0.0; + } + + // Transform E & B (lab to boosted frame) + // (Require both to for the lorentz transform) + // RHS m_parser + Ez_ext_boost = Ez_ext_lab; + Bz_ext_boost = Bz_ext_lab; + Ex_ext_boost = gamma_boost*(Ex_ext_lab - beta_boost*PhysConst::c*By_ext_lab); + Ey_ext_boost = gamma_boost*(Ey_ext_lab + beta_boost*PhysConst::c*Bx_ext_lab); + Bx_ext_boost = gamma_boost*(Bx_ext_lab + beta_boost*Ey_ext_lab/PhysConst::c); + By_ext_boost = gamma_boost*(By_ext_lab - beta_boost*Ex_ext_lab/PhysConst::c); + + // Then add to Nodal quantities in the boosted frame: + Ex_Nodal += Ex_ext_boost; + Ey_Nodal += Ey_ext_boost; + Ez_Nodal += Ez_ext_boost; + Bx_Nodal += Bx_ext_boost; + By_Nodal += By_ext_boost; + Bz_Nodal += Bz_ext_boost; + } + } else { + + // Added external e fields: + if ( external_e_fields ){ +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + Ex_Nodal += Exfield_parser(x, y, z, t); + Ey_Nodal += Eyfield_parser(x, y, z, t); + Ez_Nodal += Ezfield_parser(x, y, z, t); + } + + // Added external b fields: + if ( external_b_fields ){ +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + Bx_Nodal += Bxfield_parser(x, y, z, t); + By_Nodal += Byfield_parser(x, y, z, t); + Bz_Nodal += Bzfield_parser(x, y, z, t); + } + } + + // Isolate U from NU + amrex::Real tmp_Ux = (NUx_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real tmp_Uy = (NUy_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real tmp_Uz = (NUz_arr(i, j, k) / N_arr(i,j,k)); + + // Enforce RZ boundary conditions +#if defined(WARPX_DIM_RZ) + if ( i == 0 ){ + Ex_Nodal = 0.0; + Ey_Nodal = 0.0; + By_Nodal = 0.0; + Bx_Nodal = 0.0; + } +#endif + + // Push the fluid momentum + UpdateMomentumHigueraCary(tmp_Ux, tmp_Uy, tmp_Uz, + Ex_Nodal, Ey_Nodal, Ez_Nodal, + Bx_Nodal, By_Nodal, Bz_Nodal, q, m, dt ); + + // Calculate NU + NUx_arr(i,j,k) = N_arr(i,j,k)*tmp_Ux; + NUy_arr(i,j,k) = N_arr(i,j,k)*tmp_Uy; + NUz_arr(i,j,k) = N_arr(i,j,k)*tmp_Uz; + } + } + ); + } +} + +void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icomp) +{ + WARPX_PROFILE("WarpXFluidContainer::DepositCharge"); + + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const amrex::Periodicity &period = geom.periodicity(); + const amrex::Real q = getCharge(); + auto const &owner_mask_rho = amrex::OwnerMask(rho, period); + + // Assertion, make sure rho is at the same location as N + AMREX_ALWAYS_ASSERT(rho.ixType().nodeCentered()); + + // Loop over and deposit charge density +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 rho_arr = rho.array(mfi); + amrex::Array4 owner_mask_rho_arr = owner_mask_rho->array(mfi); + + // Deposit Rho + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + if ( owner_mask_rho_arr(i,j,k) ) rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); + } + ); + } +} + + +void WarpXFluidContainer::DepositCurrent( + int lev, + amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz) +{ + WARPX_PROFILE("WarpXFluidContainer::DepositCurrent"); + + // Temporary nodal currents + amrex::MultiFab tmp_jx_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jy_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jz_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + + const amrex::Real inv_clight_sq = 1.0_prt / PhysConst::c / PhysConst::c; + const amrex::Real q = getCharge(); + + // Prepare interpolation of current components to cell center + amrex::GpuArray j_nodal_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray jx_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray jy_type = amrex::GpuArray{0, 0, 0}; + amrex::GpuArray jz_type = amrex::GpuArray{0, 0, 0}; + for (int i = 0; i < AMREX_SPACEDIM; ++i) + { + j_nodal_type[i] = tmp_jx_fluid.ixType()[i]; + jx_type[i] = jx.ixType()[i]; + jy_type[i] = jy.ixType()[i]; + jz_type[i] = jz.ixType()[i]; + } + + // We now need to create a mask to fix the double counting. + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const amrex::Periodicity &period = geom.periodicity(); + auto const &owner_mask_x = amrex::OwnerMask(jx, period); + auto const &owner_mask_y = amrex::OwnerMask(jy, period); + auto const &owner_mask_z = amrex::OwnerMask(jz, period); + + // Calculate j at the nodes +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); + amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); + amrex::Array4 tmp_jz_fluid_arr = tmp_jz_fluid.array(mfi); + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + // Calculate J from fluid quantities + amrex::Real gamma = 1.0, Ux = 0.0, Uy = 0.0, Uz = 0.0; + if (N_arr(i, j, k)>0.0){ + Ux = NUx_arr(i, j, k)/N_arr(i, j, k); + Uy = NUy_arr(i, j, k)/N_arr(i, j, k); + Uz = NUz_arr(i, j, k)/N_arr(i, j, k); + gamma = std::sqrt(1.0 + ( Ux*Ux + Uy*Uy + Uz*Uz) * inv_clight_sq ) ; + } + tmp_jx_fluid_arr(i, j, k) = q * (NUx_arr(i, j, k) / gamma); + tmp_jy_fluid_arr(i, j, k) = q * (NUy_arr(i, j, k) / gamma); + tmp_jz_fluid_arr(i, j, k) = q * (NUz_arr(i, j, k) / gamma); + } + ); + } + + // Interpolate j from the nodes to the simulation mesh (typically Yee mesh) +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Box const &tile_box_x = mfi.tilebox(jx.ixType().toIntVect()); + amrex::Box const &tile_box_y = mfi.tilebox(jy.ixType().toIntVect()); + amrex::Box const &tile_box_z = mfi.tilebox(jz.ixType().toIntVect()); + + amrex::Array4 jx_arr = jx.array(mfi); + amrex::Array4 jy_arr = jy.array(mfi); + amrex::Array4 jz_arr = jz.array(mfi); + + amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); + amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); + amrex::Array4 tmp_jz_fluid_arr = tmp_jz_fluid.array(mfi); + + amrex::Array4 owner_mask_x_arr = owner_mask_x->array(mfi); + amrex::Array4 owner_mask_y_arr = owner_mask_y->array(mfi); + amrex::Array4 owner_mask_z_arr = owner_mask_z->array(mfi); + + // When using the `Interp` function, one needs to specify whether coarsening is desired. + // Here, we do not perform any coarsening. + amrex::GpuArray coarsening_ratio = {1, 1, 1}; + + + // Interpolate fluid current and deposit it + // ( mask double counting ) + amrex::ParallelFor( tile_box_x, tile_box_y, tile_box_z, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real jx_tmp = ablastr::coarsen::sample::Interp(tmp_jx_fluid_arr, + j_nodal_type, jx_type, coarsening_ratio, i, j, k, 0); + if ( owner_mask_x_arr(i,j,k) ) jx_arr(i, j, k) += jx_tmp; + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real jy_tmp = ablastr::coarsen::sample::Interp(tmp_jy_fluid_arr, + j_nodal_type, jy_type, coarsening_ratio, i, j, k, 0); + if ( owner_mask_y_arr(i,j,k) ) jy_arr(i, j, k) += jy_tmp; + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real jz_tmp = ablastr::coarsen::sample::Interp(tmp_jz_fluid_arr, + j_nodal_type, jz_type, coarsening_ratio, i, j, k, 0); + if ( owner_mask_z_arr(i,j,k) ) jz_arr(i, j, k) += jz_tmp; + } + ); + } +} diff --git a/Source/Fluids/WarpXFluidContainer_fwd.H b/Source/Fluids/WarpXFluidContainer_fwd.H new file mode 100644 index 00000000000..77a6bc0c991 --- /dev/null +++ b/Source/Fluids/WarpXFluidContainer_fwd.H @@ -0,0 +1,12 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_WarpXFluidContainer_fwd_H_ + +class WarpXFluidContainer; + +#endif /* WARPX_WarpXFluidContainer_fwd_H_ */ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 87b81381ce9..b0afafa730b 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -94,6 +94,7 @@ namespace { * @param u_th Momentum spread * @param engine Object used to generate random numbers */ + AMREX_FORCE_INLINE AMREX_GPU_HOST_DEVICE amrex::Real generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 89ea4a00a78..ac64980b952 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -120,11 +120,7 @@ public: bool radially_weighted = true; - std::string str_density_function; std::string str_flux_function; - std::string str_momentum_function_ux; - std::string str_momentum_function_uy; - std::string str_momentum_function_uz; amrex::Real xmin, xmax; amrex::Real ymin, ymax; @@ -146,7 +142,6 @@ protected: PhysicalSpecies physical_species = PhysicalSpecies::unspecified; - amrex::Real density; amrex::Real flux; int species_id; @@ -185,9 +180,7 @@ protected: void setupNuniformPerCell (const amrex::ParmParse& pp_species_name); void setupExternalFile (const amrex::ParmParse& pp_species_name); - void parseDensity (const amrex::ParmParse& pp_species_name); void parseFlux (const amrex::ParmParse& pp_species_name); - void parseMomentum (const amrex::ParmParse& pp_species_name, const std::string& style); }; #endif diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index b880d4a46e7..56d29a8a1bb 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -14,8 +14,8 @@ #include "Initialization/InjectorDensity.H" #include "Initialization/InjectorMomentum.H" #include "Initialization/InjectorPosition.H" -#include "Particles/SpeciesPhysicalProperties.H" #include "Utils/Parser/ParserUtils.H" +#include "Utils/SpeciesUtils.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" @@ -45,17 +45,6 @@ using namespace amrex::literals; -namespace { - void StringParseAbortMessage(const std::string& var, - const std::string& name) { - std::stringstream stringstream; - std::string string; - stringstream << var << " string '" << name << "' not recognized."; - string = stringstream.str(); - WARPX_ABORT_WITH_MESSAGE(string); - } -} - PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom): species_id{ispecies}, species_name{name} @@ -123,60 +112,14 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, utils::parser::queryWithParser(pp_species_name, "density_min", density_min); utils::parser::queryWithParser(pp_species_name, "density_max", density_max); - std::string physical_species_s; - const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); - if (species_is_specified){ - const auto physical_species_from_string = species::from_string( physical_species_s ); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, - physical_species_s + " does not exist!"); - physical_species = physical_species_from_string.value(); - charge = species::get_charge( physical_species ); - mass = species::get_mass( physical_species ); - } - - // Parse injection style std::string injection_style = "none"; + // Parse injection style pp_species_name.query("injection_style", injection_style); std::transform(injection_style.begin(), - injection_style.end(), - injection_style.begin(), - ::tolower); - - // parse charge and mass - const bool charge_is_specified = - utils::parser::queryWithParser(pp_species_name, "charge", charge); - const bool mass_is_specified = - utils::parser::queryWithParser(pp_species_name, "mass", mass); - - if ( charge_is_specified && species_is_specified ){ - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + species_name + ".charge' and " + - species_name + ".species_type' are specified.\n" + - species_name + ".charge' will take precedence.\n"); - - } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - charge_is_specified || - species_is_specified || - (injection_style == "external_file"), - "Need to specify at least one of species_type or charge for species '" + - species_name + "'." - ); - - if ( mass_is_specified && species_is_specified ){ - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + species_name + ".mass' and " + - species_name + ".species_type' are specified.\n" + - species_name + ".mass' will take precedence.\n"); - } - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - mass_is_specified || - species_is_specified || - (injection_style == "external_file"), - "Need to specify at least one of species_type or mass for species '" + - species_name + "'." - ); + injection_style.end(), + injection_style.begin(), + ::tolower); + SpeciesUtils::extractSpeciesProperties(species_name, injection_style, charge, mass, physical_species); num_particles_per_cell_each_dim.assign(3, 0); @@ -197,9 +140,27 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, } else if (injection_style == "external_file") { setupExternalFile(pp_species_name); } else if (injection_style != "none") { - StringParseAbortMessage("Injection style", injection_style); + SpeciesUtils::StringParseAbortMessage("Injection style", injection_style); } + if (h_inj_rho) { +#ifdef AMREX_USE_GPU + d_inj_rho = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorDensity))); + amrex::Gpu::htod_memcpy_async(d_inj_rho, h_inj_rho.get(), sizeof(InjectorDensity)); +#else + d_inj_rho = h_inj_rho.get(); +#endif + } + if (h_inj_mom) { +#ifdef AMREX_USE_GPU + d_inj_mom = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorMomentum))); + amrex::Gpu::htod_memcpy_async(d_inj_mom, h_inj_mom.get(), sizeof(InjectorMomentum)); +#else + d_inj_mom = h_inj_mom.get(); +#endif + } amrex::Gpu::synchronize(); } @@ -286,7 +247,8 @@ void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( valid_symmetries.count(symmetrization_order), "Error: Symmetrization only supported to orders 4 or 8 "); gaussian_beam = true; - parseMomentum(pp_species_name, "gaussian_beam"); + SpeciesUtils::parseMomentum(species_name, "gaussian_beam", h_inj_mom, + ux_parser, uy_parser, uz_parser, h_mom_temp, h_mom_vel); #if defined(WARPX_DIM_XZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( y_rms > 0._rt, "Error: Gaussian beam y_rms must be strictly greater than 0 in 2D " @@ -325,9 +287,9 @@ void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_nam #else d_inj_pos = h_inj_pos.get(); #endif - - parseDensity(pp_species_name); - parseMomentum(pp_species_name, "nrandompercell"); + SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, "nrandompercell", h_inj_mom, + ux_parser, uy_parser, uz_parser, h_mom_temp, h_mom_vel); } void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) @@ -404,7 +366,8 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) #endif parseFlux(pp_species_name); - parseMomentum(pp_species_name, "nfluxpercell"); + SpeciesUtils::parseMomentum(species_name, "nfluxpercell", h_inj_mom, + ux_parser, uy_parser, uz_parser, h_mom_temp, h_mom_vel, flux_normal_axis, flux_direction); } void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_name) @@ -455,8 +418,9 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na num_particles_per_cell = num_particles_per_cell_each_dim[0] * num_particles_per_cell_each_dim[1] * num_particles_per_cell_each_dim[2]; - parseDensity(pp_species_name); - parseMomentum(pp_species_name, "nuniformpercell"); + SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, "nuniformpercell", h_inj_mom, + ux_parser, uy_parser, uz_parser, h_mom_temp, h_mom_vel); } void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) @@ -563,46 +527,6 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) #endif // WARPX_USE_OPENPMD } -// Depending on injection type at runtime, initialize inj_rho -// so that inj_rho->getDensity calls -// InjectorPosition[Constant or Predefined or etc.].getDensity. -void PlasmaInjector::parseDensity (const amrex::ParmParse& pp_species_name) -{ - // parse density information - std::string rho_prof_s; - pp_species_name.get("profile", rho_prof_s); - std::transform(rho_prof_s.begin(), rho_prof_s.end(), - rho_prof_s.begin(), ::tolower); - if (rho_prof_s == "constant") { - utils::parser::getWithParser(pp_species_name, "density", density); - // Construct InjectorDensity with InjectorDensityConstant. - h_inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); - } else if (rho_prof_s == "predefined") { - // Construct InjectorDensity with InjectorDensityPredefined. - h_inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); - } else if (rho_prof_s == "parse_density_function") { - utils::parser::Store_parserString( - pp_species_name, "density_function(x,y,z)", str_density_function); - // Construct InjectorDensity with InjectorDensityParser. - density_parser = std::make_unique( - utils::parser::makeParser(str_density_function,{"x","y","z"})); - h_inj_rho.reset(new InjectorDensity((InjectorDensityParser*)nullptr, - density_parser->compile<3>())); - } else { - StringParseAbortMessage("Density profile type", rho_prof_s); - } - - if (h_inj_rho) { -#ifdef AMREX_USE_GPU - d_inj_rho = static_cast - (amrex::The_Arena()->alloc(sizeof(InjectorDensity))); - amrex::Gpu::htod_memcpy_async(d_inj_rho, h_inj_rho.get(), sizeof(InjectorDensity)); -#else - d_inj_rho = h_inj_rho.get(); -#endif - } -} - // Depending on injection type at runtime, initialize inj_flux // so that inj_flux->getFlux calls // InjectorFlux[Constant or Parser or etc.].getFlux. @@ -626,7 +550,7 @@ void PlasmaInjector::parseFlux (const amrex::ParmParse& pp_species_name) h_inj_flux.reset(new InjectorFlux((InjectorFluxParser*)nullptr, flux_parser->compile<4>())); } else { - StringParseAbortMessage("Flux profile type", flux_prof_s); + SpeciesUtils::StringParseAbortMessage("Flux profile type", flux_prof_s); } if (h_inj_flux) { #ifdef AMREX_USE_GPU @@ -640,138 +564,6 @@ void PlasmaInjector::parseFlux (const amrex::ParmParse& pp_species_name) } -// Depending on injection type at runtime, initialize inj_mom -// so that inj_mom->getMomentum calls -// InjectorMomentum[Constant or Gaussian or etc.].getMomentum. -void PlasmaInjector::parseMomentum (const amrex::ParmParse& pp_species_name, const std::string& style) -{ - using namespace amrex::literals; - - // parse momentum information - std::string mom_dist_s; - pp_species_name.get("momentum_distribution_type", mom_dist_s); - std::transform(mom_dist_s.begin(), - mom_dist_s.end(), - mom_dist_s.begin(), - ::tolower); - if (mom_dist_s == "at_rest") { - constexpr amrex::Real ux = 0._rt; - constexpr amrex::Real uy = 0._rt; - constexpr amrex::Real uz = 0._rt; - // Construct InjectorMomentum with InjectorMomentumConstant. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); - } else if (mom_dist_s == "constant") { - amrex::Real ux = 0._rt; - amrex::Real uy = 0._rt; - amrex::Real uz = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux", ux); - utils::parser::queryWithParser(pp_species_name, "uy", uy); - utils::parser::queryWithParser(pp_species_name, "uz", uz); - // Construct InjectorMomentum with InjectorMomentumConstant. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); - } else if (mom_dist_s == "gaussian") { - amrex::Real ux_m = 0._rt; - amrex::Real uy_m = 0._rt; - amrex::Real uz_m = 0._rt; - amrex::Real ux_th = 0._rt; - amrex::Real uy_th = 0._rt; - amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); - // Construct InjectorMomentum with InjectorMomentumGaussian. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, - ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); - } else if (mom_dist_s == "gaussianflux") { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(style == "nfluxpercell", - "Error: gaussianflux can only be used with injection_style = NFluxPerCell"); - amrex::Real ux_m = 0._rt; - amrex::Real uy_m = 0._rt; - amrex::Real uz_m = 0._rt; - amrex::Real ux_th = 0._rt; - amrex::Real uy_th = 0._rt; - amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); - // Construct InjectorMomentum with InjectorMomentumGaussianFlux. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianFlux*)nullptr, - ux_m, uy_m, uz_m, ux_th, uy_th, uz_th, - flux_normal_axis, flux_direction)); - } else if (mom_dist_s == "uniform") { - amrex::Real ux_min = 0._rt; - amrex::Real uy_min = 0._rt; - amrex::Real uz_min = 0._rt; - amrex::Real ux_max = 0._rt; - amrex::Real uy_max = 0._rt; - amrex::Real uz_max = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_min", ux_min); - utils::parser::queryWithParser(pp_species_name, "uy_min", uy_min); - utils::parser::queryWithParser(pp_species_name, "uz_min", uz_min); - utils::parser::queryWithParser(pp_species_name, "ux_max", ux_max); - utils::parser::queryWithParser(pp_species_name, "uy_max", uy_max); - utils::parser::queryWithParser(pp_species_name, "uz_max", uz_max); - // Construct InjectorMomentum with InjectorMomentumUniform. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumUniform*)nullptr, - ux_min, uy_min, uz_min, ux_max, uy_max, uz_max)); - } else if (mom_dist_s == "maxwell_boltzmann"){ - h_mom_temp = std::make_unique(pp_species_name); - const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); - const GetVelocity getVel(*h_mom_vel); - // Construct InjectorMomentum with InjectorMomentumBoltzmann. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, getTemp, getVel)); - } else if (mom_dist_s == "maxwell_juttner"){ - h_mom_temp = std::make_unique(pp_species_name); - const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); - const GetVelocity getVel(*h_mom_vel); - // Construct InjectorMomentum with InjectorMomentumJuttner. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, getTemp, getVel)); - } else if (mom_dist_s == "radial_expansion") { - amrex::Real u_over_r = 0._rt; - utils::parser::queryWithParser(pp_species_name, "u_over_r", u_over_r); - // Construct InjectorMomentum with InjectorMomentumRadialExpansion. - h_inj_mom.reset(new InjectorMomentum - ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); - } else if (mom_dist_s == "parse_momentum_function") { - utils::parser::Store_parserString(pp_species_name, "momentum_function_ux(x,y,z)", - str_momentum_function_ux); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uy(x,y,z)", - str_momentum_function_uy); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uz(x,y,z)", - str_momentum_function_uz); - // Construct InjectorMomentum with InjectorMomentumParser. - ux_parser = std::make_unique( - utils::parser::makeParser(str_momentum_function_ux, {"x","y","z"})); - uy_parser = std::make_unique( - utils::parser::makeParser(str_momentum_function_uy, {"x","y","z"})); - uz_parser = std::make_unique( - utils::parser::makeParser(str_momentum_function_uz, {"x","y","z"})); - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumParser*)nullptr, - ux_parser->compile<3>(), - uy_parser->compile<3>(), - uz_parser->compile<3>())); - } else { - StringParseAbortMessage("Momentum distribution type", mom_dist_s); - } - if (h_inj_mom) { -#ifdef AMREX_USE_GPU - d_inj_mom = static_cast - (amrex::The_Arena()->alloc(sizeof(InjectorMomentum))); - amrex::Gpu::htod_memcpy_async(d_inj_mom, h_inj_mom.get(), sizeof(InjectorMomentum)); -#else - d_inj_mom = h_inj_mom.get(); -#endif - } -} - amrex::XDim3 PlasmaInjector::getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 824c16df5b8..eb44ed4eae5 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -77,6 +77,7 @@ include $(WARPX_HOME)/Source/Diagnostics/Make.package include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package include $(WARPX_HOME)/Source/FieldSolver/Make.package include $(WARPX_HOME)/Source/Filter/Make.package +include $(WARPX_HOME)/Source/Fluids/Make.package include $(WARPX_HOME)/Source/Initialization/Make.package include $(WARPX_HOME)/Source/Laser/Make.package include $(WARPX_HOME)/Source/Parallelization/Make.package diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H index 7c2432f6e1b..3101047e211 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H @@ -102,184 +102,193 @@ void UpdateMomentumPerezElastic ( T_PR const g1s = ( T_PR(1.0) - vcDv1*inv_c2 )*gc*g1; T_PR const g2s = ( T_PR(1.0) - vcDv2*inv_c2 )*gc*g2; - // Compute the Coulomb log lnLmd - T_PR lnLmd; - if ( L > T_PR(0.0) ) { lnLmd = L; } - else - { - // Compute b0 according to eq (22) from Perez et al., Phys.Plasmas.19.083104 (2012) - // Note: there is a typo in the equation, the last square is incorrect! - // See the SMILEI documentation: https://smileipic.github.io/Smilei/Understand/collisions.html - // and https://github.com/ECP-WarpX/WarpX/files/3799803/main.pdf from GitHub #429 - T_PR const b0 = amrex::Math::abs(q1*q2) * inv_c2 / - (T_PR(4.0)*MathConst::pi*PhysConst::ep0) * gc/mass_g * - ( m1*g1s*m2*g2s/(p1sm*p1sm*inv_c2) + T_PR(1.0) ); - - // Compute the minimal impact parameter - constexpr T_PR hbar_pi = static_cast(PhysConst::hbar*MathConst::pi); - const T_PR bmin = amrex::max(hbar_pi/p1sm, b0); - - // Compute the Coulomb log lnLmd - lnLmd = amrex::max( T_PR(2.0), - T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); - } - // Compute s - const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); - const auto tts2 = tts*tts; - T_PR s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / - ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * - m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; - - // Compute s' - const auto cbrt_n1 = std::cbrt(n1); - const auto cbrt_n2 = std::cbrt(n2); - const auto coeff = static_cast( - std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); - T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); - T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / - amrex::max( m1*cbrt_n1*cbrt_n1, - m2*cbrt_n2*cbrt_n2); - - // Determine s - s = amrex::min(s,sp); - - // Get random numbers - T_PR r = amrex::Random(engine); - - // Compute scattering angle - T_PR cosXs; - T_PR sinXs; - if ( s <= T_PR(0.1) ) - { - while ( true ) + T_PR s = 0; + if (p1sm > std::numeric_limits::min()) { + + // s is non-zero (i.e. particles scatter) only if the relative + // motion between particles is not negligible (p1sm non-zero) + + // Compute the Coulomb log lnLmd first + T_PR lnLmd; + if ( L > T_PR(0.0) ) { lnLmd = L; } + else { - cosXs = T_PR(1.0) + s * std::log(r); - // Avoid the bug when r is too small such that cosXs < -1 - if ( cosXs >= T_PR(-1.0) ) { break; } - r = amrex::Random(engine); + // Compute b0 according to eq (22) from Perez et al., Phys.Plasmas.19.083104 (2012) + // Note: there is a typo in the equation, the last square is incorrect! + // See the SMILEI documentation: https://smileipic.github.io/Smilei/Understand/collisions.html + // and https://github.com/ECP-WarpX/WarpX/files/3799803/main.pdf from GitHub #429 + T_PR const b0 = amrex::Math::abs(q1*q2) * inv_c2 / + (T_PR(4.0)*MathConst::pi*PhysConst::ep0) * gc/mass_g * + ( m1*g1s*m2*g2s/(p1sm*p1sm*inv_c2) + T_PR(1.0) ); + + // Compute the minimal impact parameter + constexpr T_PR hbar_pi = static_cast(PhysConst::hbar*MathConst::pi); + const T_PR bmin = amrex::max(hbar_pi/p1sm, b0); + + // Compute the Coulomb log lnLmd + lnLmd = amrex::max( T_PR(2.0), + T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); } - } - else if ( s > T_PR(0.1) && s <= T_PR(3.0) ) - { - T_PR const Ainv = static_cast( - 0.0056958 + 0.9560202*s - 0.508139*s*s + - 0.47913906*s*s*s - 0.12788975*s*s*s*s + 0.02389567*s*s*s*s*s); - cosXs = Ainv * std::log( std::exp(T_PR(-1.0)/Ainv) + - T_PR(2.0) * r * std::sinh(T_PR(1.0)/Ainv) ); - } - else if ( s > T_PR(3.0) && s <= T_PR(6.0) ) - { - T_PR const A = T_PR(3.0) * std::exp(-s); - cosXs = T_PR(1.0)/A * std::log( std::exp(-A) + - T_PR(2.0) * r * std::sinh(A) ); - } - else - { - cosXs = T_PR(2.0) * r - T_PR(1.0); - } - sinXs = std::sqrt(T_PR(1.0) - cosXs*cosXs); - - // Get random azimuthal angle - T_PR const phis = amrex::Random(engine) * T_PR(2.0) * MathConst::pi; - T_PR const cosphis = std::cos(phis); - T_PR const sinphis = std::sin(phis); - - // Compute post-collision momenta pfs in COM - T_PR p1fsx; - T_PR p1fsy; - T_PR p1fsz; - // p1sp is the p1s perpendicular - T_PR p1sp = std::sqrt( p1sx*p1sx + p1sy*p1sy ); - // Make sure p1sp is not almost zero - if ( p1sp > std::numeric_limits::min() ) - { - p1fsx = ( p1sx*p1sz/p1sp ) * sinXs*cosphis + - ( p1sy*p1sm/p1sp ) * sinXs*sinphis + - ( p1sx ) * cosXs; - p1fsy = ( p1sy*p1sz/p1sp ) * sinXs*cosphis + - (-p1sx*p1sm/p1sp ) * sinXs*sinphis + - ( p1sy ) * cosXs; - p1fsz = (-p1sp ) * sinXs*cosphis + - ( T_PR(0.0) ) * sinXs*sinphis + - ( p1sz ) * cosXs; - // Note a negative sign is different from - // Eq. (12) in Perez's paper, - // but they are the same due to the random nature of phis. - } - else - { - // If the previous p1sp is almost zero - // x->y y->z z->x - // This set is equivalent to the one in Nanbu's paper - p1sp = std::sqrt( p1sy*p1sy + p1sz*p1sz ); - p1fsy = ( p1sy*p1sx/p1sp ) * sinXs*cosphis + - ( p1sz*p1sm/p1sp ) * sinXs*sinphis + - ( p1sy ) * cosXs; - p1fsz = ( p1sz*p1sx/p1sp ) * sinXs*cosphis + - (-p1sy*p1sm/p1sp ) * sinXs*sinphis + - ( p1sz ) * cosXs; - p1fsx = (-p1sp ) * sinXs*cosphis + - ( T_PR(0.0) ) * sinXs*sinphis + - ( p1sx ) * cosXs; - } - T_PR const p2fsx = -p1fsx; - T_PR const p2fsy = -p1fsy; - T_PR const p2fsz = -p1fsz; + // Compute s + const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); + const auto tts2 = tts*tts; + s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / + ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * + m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; - // Transform from COM to lab frame - T_PR p1fx; T_PR p2fx; - T_PR p1fy; T_PR p2fy; - T_PR p1fz; T_PR p2fz; - if ( vcms > std::numeric_limits::min() ) - { - T_PR const vcDp1fs = vcx*p1fsx + vcy*p1fsy + vcz*p1fsz; - T_PR const vcDp2fs = vcx*p2fsx + vcy*p2fsy + vcz*p2fsz; - /* factor = (gc-1.0)/vcms; Rewrite to avoid subtraction losing precision when gc is close to 1 */ - T_PR const factor = gc*gc*inv_c2/(gc+T_PR(1.0)); - T_PR const factor1 = factor*vcDp1fs + m1*g1s*gc; - T_PR const factor2 = factor*vcDp2fs + m2*g2s*gc; - p1fx = p1fsx + vcx * factor1; - p1fy = p1fsy + vcy * factor1; - p1fz = p1fsz + vcz * factor1; - p2fx = p2fsx + vcx * factor2; - p2fy = p2fsy + vcy * factor2; - p2fz = p2fsz + vcz * factor2; - } - else // If vcms = 0, don't do Lorentz-transform. - { - p1fx = p1fsx; - p1fy = p1fsy; - p1fz = p1fsz; - p2fx = p2fsx; - p2fy = p2fsy; - p2fz = p2fsz; - } + // Compute s' + const auto cbrt_n1 = std::cbrt(n1); + const auto cbrt_n2 = std::cbrt(n2); + const auto coeff = static_cast( + std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); + T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); + T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / + amrex::max( m1*cbrt_n1*cbrt_n1, + m2*cbrt_n2*cbrt_n2); - // Rejection method - r = amrex::Random(engine); - if ( w2 > r*amrex::max(w1, w2) ) - { - u1x = p1fx / m1; - u1y = p1fy / m1; - u1z = p1fz / m1; -#ifndef AMREX_USE_DPCPP - AMREX_ASSERT(!std::isnan(u1x+u1y+u1z+u2x+u2y+u2z)); - AMREX_ASSERT(!std::isinf(u1x+u1y+u1z+u2x+u2y+u2z)); -#endif + // Determine s + s = amrex::min(s,sp); } - r = amrex::Random(engine); - if ( w1 > r*amrex::max(w1, w2) ) - { - u2x = p2fx / m2; - u2y = p2fy / m2; - u2z = p2fz / m2; + + // Only modify momenta if is s is non-zero + if (s > std::numeric_limits::min()) { + + // Get random numbers + T_PR r = amrex::Random(engine); + + // Compute scattering angle + T_PR cosXs; + T_PR sinXs; + if ( s <= T_PR(0.1) ) + { + while ( true ) + { + cosXs = T_PR(1.0) + s * std::log(r); + // Avoid the bug when r is too small such that cosXs < -1 + if ( cosXs >= T_PR(-1.0) ) { break; } + r = amrex::Random(engine); + } + } + else if ( s > T_PR(0.1) && s <= T_PR(3.0) ) + { + T_PR const Ainv = static_cast( + 0.0056958 + 0.9560202*s - 0.508139*s*s + + 0.47913906*s*s*s - 0.12788975*s*s*s*s + 0.02389567*s*s*s*s*s); + cosXs = Ainv * std::log( std::exp(T_PR(-1.0)/Ainv) + + T_PR(2.0) * r * std::sinh(T_PR(1.0)/Ainv) ); + } + else if ( s > T_PR(3.0) && s <= T_PR(6.0) ) + { + T_PR const A = T_PR(3.0) * std::exp(-s); + cosXs = T_PR(1.0)/A * std::log( std::exp(-A) + + T_PR(2.0) * r * std::sinh(A) ); + } + else + { + cosXs = T_PR(2.0) * r - T_PR(1.0); + } + sinXs = std::sqrt(T_PR(1.0) - cosXs*cosXs); + + // Get random azimuthal angle + T_PR const phis = amrex::Random(engine) * T_PR(2.0) * MathConst::pi; + T_PR const cosphis = std::cos(phis); + T_PR const sinphis = std::sin(phis); + + // Compute post-collision momenta pfs in COM + T_PR p1fsx; + T_PR p1fsy; + T_PR p1fsz; + // p1sp is the p1s perpendicular + T_PR p1sp = std::sqrt( p1sx*p1sx + p1sy*p1sy ); + // Make sure p1sp is not almost zero + if ( p1sp > std::numeric_limits::min() ) + { + p1fsx = ( p1sx*p1sz/p1sp ) * sinXs*cosphis + + ( p1sy*p1sm/p1sp ) * sinXs*sinphis + + ( p1sx ) * cosXs; + p1fsy = ( p1sy*p1sz/p1sp ) * sinXs*cosphis + + (-p1sx*p1sm/p1sp ) * sinXs*sinphis + + ( p1sy ) * cosXs; + p1fsz = (-p1sp ) * sinXs*cosphis + + ( T_PR(0.0) ) * sinXs*sinphis + + ( p1sz ) * cosXs; + // Note a negative sign is different from + // Eq. (12) in Perez's paper, + // but they are the same due to the random nature of phis. + } + else + { + // If the previous p1sp is almost zero + // x->y y->z z->x + // This set is equivalent to the one in Nanbu's paper + p1sp = std::sqrt( p1sy*p1sy + p1sz*p1sz ); + p1fsy = ( p1sy*p1sx/p1sp ) * sinXs*cosphis + + ( p1sz*p1sm/p1sp ) * sinXs*sinphis + + ( p1sy ) * cosXs; + p1fsz = ( p1sz*p1sx/p1sp ) * sinXs*cosphis + + (-p1sy*p1sm/p1sp ) * sinXs*sinphis + + ( p1sz ) * cosXs; + p1fsx = (-p1sp ) * sinXs*cosphis + + ( T_PR(0.0) ) * sinXs*sinphis + + ( p1sx ) * cosXs; + } + + T_PR const p2fsx = -p1fsx; + T_PR const p2fsy = -p1fsy; + T_PR const p2fsz = -p1fsz; + + // Transform from COM to lab frame + T_PR p1fx; T_PR p2fx; + T_PR p1fy; T_PR p2fy; + T_PR p1fz; T_PR p2fz; + if ( vcms > std::numeric_limits::min() ) + { + T_PR const vcDp1fs = vcx*p1fsx + vcy*p1fsy + vcz*p1fsz; + T_PR const vcDp2fs = vcx*p2fsx + vcy*p2fsy + vcz*p2fsz; + /* factor = (gc-1.0)/vcms; Rewrite to avoid subtraction losing precision when gc is close to 1 */ + T_PR const factor = gc*gc*inv_c2/(gc+T_PR(1.0)); + T_PR const factor1 = factor*vcDp1fs + m1*g1s*gc; + T_PR const factor2 = factor*vcDp2fs + m2*g2s*gc; + p1fx = p1fsx + vcx * factor1; + p1fy = p1fsy + vcy * factor1; + p1fz = p1fsz + vcz * factor1; + p2fx = p2fsx + vcx * factor2; + p2fy = p2fsy + vcy * factor2; + p2fz = p2fsz + vcz * factor2; + } + else // If vcms = 0, don't do Lorentz-transform. + { + p1fx = p1fsx; + p1fy = p1fsy; + p1fz = p1fsz; + p2fx = p2fsx; + p2fy = p2fsy; + p2fz = p2fsz; + } + + // Rejection method + r = amrex::Random(engine); + if ( w2 > r*amrex::max(w1, w2) ) + { + u1x = p1fx / m1; + u1y = p1fy / m1; + u1z = p1fz / m1; + } + r = amrex::Random(engine); + if ( w1 > r*amrex::max(w1, w2) ) + { + u2x = p2fx / m2; + u2y = p2fy / m2; + u2z = p2fz / m2; + } #ifndef AMREX_USE_DPCPP AMREX_ASSERT(!std::isnan(u1x+u1y+u1z+u2x+u2y+u2z)); AMREX_ASSERT(!std::isinf(u1x+u1y+u1z+u2x+u2y+u2z)); #endif - } + + } // if s > std::numeric_limits::min() } diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index fecd39137d7..af826068806 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -79,11 +79,6 @@ public: WarpXParticleContainer& GetParticleContainerFromName (const std::string& name) const; -#ifdef WARPX_USE_OPENPMD - std::unique_ptr& GetUniqueContainer(int index) { - return allcontainers[index]; - } -#endif std::array meanParticleVelocity(int index) { return allcontainers[index]->meanParticleVelocity(); } diff --git a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H index 359fc34d508..dc998caee34 100644 --- a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H +++ b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H @@ -17,49 +17,49 @@ * \brief Push the particle's positions over one timestep, * given the value of its momenta `ux`, `uy`, `uz` */ - +template AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumHigueraCary( - amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, - const amrex::ParticleReal Ex, const amrex::ParticleReal Ey, const amrex::ParticleReal Ez, - const amrex::ParticleReal Bx, const amrex::ParticleReal By, const amrex::ParticleReal Bz, - const amrex::ParticleReal q, const amrex::ParticleReal m, const amrex::Real dt ) + T& ux, T& uy, T& uz, + const T Ex, const T Ey, const T Ez, + const T Bx, const T By, const T Bz, + const T q, const T m, const amrex::Real dt ) { using namespace amrex::literals; // Constants - const amrex::ParticleReal qmt = 0.5_prt*q*dt/m; - constexpr amrex::ParticleReal invclight = 1._prt/PhysConst::c; - constexpr amrex::ParticleReal invclightsq = 1._prt/(PhysConst::c*PhysConst::c); + const T qmt = 0.5_prt*q*dt/m; + constexpr T invclight = 1._prt/PhysConst::c; + constexpr T invclightsq = 1._prt/(PhysConst::c*PhysConst::c); // Compute u_minus - const amrex::ParticleReal umx = ux + qmt*Ex; - const amrex::ParticleReal umy = uy + qmt*Ey; - const amrex::ParticleReal umz = uz + qmt*Ez; + const T umx = ux + qmt*Ex; + const T umy = uy + qmt*Ey; + const T umz = uz + qmt*Ez; // Compute gamma squared of u_minus - amrex::ParticleReal gamma = 1._prt + (umx*umx + umy*umy + umz*umz)*invclightsq; + T gamma = 1._prt + (umx*umx + umy*umy + umz*umz)*invclightsq; // Compute beta and betam squared - const amrex::ParticleReal betax = qmt*Bx; - const amrex::ParticleReal betay = qmt*By; - const amrex::ParticleReal betaz = qmt*Bz; - const amrex::ParticleReal betam = betax*betax + betay*betay + betaz*betaz; + const T betax = qmt*Bx; + const T betay = qmt*By; + const T betaz = qmt*Bz; + const T betam = betax*betax + betay*betay + betaz*betaz; // Compute sigma - const amrex::ParticleReal sigma = gamma - betam; + const T sigma = gamma - betam; // Get u* - const amrex::ParticleReal ust = (umx*betax + umy*betay + umz*betaz)*invclight; + const T ust = (umx*betax + umy*betay + umz*betaz)*invclight; // Get new gamma inversed gamma = 1._prt/std::sqrt(0.5_prt*(sigma + std::sqrt(sigma*sigma + 4._prt*(betam + ust*ust)) )); // Compute t - const amrex::ParticleReal tx = gamma*betax; - const amrex::ParticleReal ty = gamma*betay; - const amrex::ParticleReal tz = gamma*betaz; + const T tx = gamma*betax; + const T ty = gamma*betay; + const T tz = gamma*betaz; // Compute s - const amrex::ParticleReal s = 1._prt/(1._prt+(tx*tx + ty*ty + tz*tz)); + const T s = 1._prt/(1._prt+(tx*tx + ty*ty + tz*tz)); // Compute um dot t - const amrex::ParticleReal umt = umx*tx + umy*ty + umz*tz; + const T umt = umx*tx + umy*ty + umz*tz; // Compute u_plus - const amrex::ParticleReal upx = s*( umx + umt*tx + umy*tz - umz*ty ); - const amrex::ParticleReal upy = s*( umy + umt*ty + umz*tx - umx*tz ); - const amrex::ParticleReal upz = s*( umz + umt*tz + umx*ty - umy*tx ); + const T upx = s*( umx + umt*tx + umy*tz - umz*ty ); + const T upy = s*( umy + umt*ty + umz*tx - umx*tz ); + const T upz = s*( umz + umt*tz + umx*ty - umy*tx ); // Get new u ux = upx + qmt*Ex + upy*tz - upz*ty; uy = upy + qmt*Ey + upz*tx - upx*tz; diff --git a/Source/Particles/Sorting/CMakeLists.txt b/Source/Particles/Sorting/CMakeLists.txt index d3c378e43b8..60a156f4649 100644 --- a/Source/Particles/Sorting/CMakeLists.txt +++ b/Source/Particles/Sorting/CMakeLists.txt @@ -3,5 +3,6 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE Partition.cpp + SortingUtils.cpp ) endforeach() diff --git a/Source/Particles/Sorting/Make.package b/Source/Particles/Sorting/Make.package index 16efc02b89e..e6ad1604dc7 100644 --- a/Source/Particles/Sorting/Make.package +++ b/Source/Particles/Sorting/Make.package @@ -1,2 +1,4 @@ CEXE_sources += Partition.cpp +CEXE_sources += SortingUtils.cpp + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Sorting diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index 0bec92d6efc..dd53ad6f610 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -19,18 +19,7 @@ * * \param[inout] v Vector of integers, to be filled by this routine */ -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) -{ -#ifdef AMREX_USE_GPU - // On GPU: Use amrex - auto data = v.data(); - auto N = v.size(); - AMREX_FOR_1D( N, i, {data[i] = i;}); -#else - // On CPU: Use std library - std::iota( v.begin(), v.end(), 0L ); -#endif -} +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); /** \brief Find the indices that would reorder the elements of `predicate` * so that the elements with non-zero value precede the other elements diff --git a/Source/Particles/Sorting/SortingUtils.cpp b/Source/Particles/Sorting/SortingUtils.cpp new file mode 100644 index 00000000000..699119e8e18 --- /dev/null +++ b/Source/Particles/Sorting/SortingUtils.cpp @@ -0,0 +1,22 @@ +/* Copyright 2019-2020 Andrew Myers, Maxence Thevenet, Remi Lehe + * Weiqun Zhang + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "SortingUtils.H" + +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) +{ +#ifdef AMREX_USE_GPU + // On GPU: Use amrex + auto data = v.data(); + auto N = v.size(); + AMREX_FOR_1D( N, i, {data[i] = i;}); +#else + // On CPU: Use std library + std::iota( v.begin(), v.end(), 0L ); +#endif +} diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index b5d3b16269c..386225c8968 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -115,5 +115,12 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("rho"), py::arg("lev") ) + .def("get_charge_density", + [](WarpXParticleContainer& pc, int lev, bool local) + { + return pc.GetChargeDensity(lev, local); + }, + py::arg("lev"), py::arg("local") + ) ; } diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 3429f33f5b9..1a5b319577c 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/Source/Utils/CMakeLists.txt b/Source/Utils/CMakeLists.txt index 9ded83a0c29..77626c0681e 100644 --- a/Source/Utils/CMakeLists.txt +++ b/Source/Utils/CMakeLists.txt @@ -4,6 +4,7 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE Interpolate.cpp ParticleUtils.cpp + SpeciesUtils.cpp RelativeCellPosition.cpp WarpXAlgorithmSelection.cpp WarpXMovingWindow.cpp diff --git a/Source/Utils/Make.package b/Source/Utils/Make.package index eb318f8d23c..4b5888ef22b 100644 --- a/Source/Utils/Make.package +++ b/Source/Utils/Make.package @@ -8,6 +8,7 @@ CEXE_sources += Interpolate.cpp CEXE_sources += IntervalsParser.cpp CEXE_sources += RelativeCellPosition.cpp CEXE_sources += ParticleUtils.cpp +CEXE_sources += SpeciesUtils.cpp include $(WARPX_HOME)/Source/Utils/Algorithms/Make.package include $(WARPX_HOME)/Source/Utils/Logo/Make.package diff --git a/Source/Utils/SpeciesUtils.H b/Source/Utils/SpeciesUtils.H new file mode 100644 index 00000000000..b1a2618a002 --- /dev/null +++ b/Source/Utils/SpeciesUtils.H @@ -0,0 +1,39 @@ +/* Copyright 2023 RemiLehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_SPECIES_UTILS_H_ +#define WARPX_SPECIES_UTILS_H_ + +#include +#include "Initialization/InjectorDensity.H" +#include "Initialization/InjectorMomentum.H" +#include "Particles/SpeciesPhysicalProperties.H" + +namespace SpeciesUtils { + + void StringParseAbortMessage(const std::string& var, + const std::string& name); + + void extractSpeciesProperties ( std::string const& species_name, + std::string const& injection_style, amrex::Real& charge, amrex::Real& mass, + PhysicalSpecies& physical_species); + + void parseDensity (std::string const& species_name, + std::unique_ptr& h_inj_rho, + std::unique_ptr& density_parser); + + void parseMomentum (std::string const& species_name, const std::string& style, + std::unique_ptr& h_inj_mom, + std::unique_ptr& ux_parser, + std::unique_ptr& uy_parser, + std::unique_ptr& uz_parser, + std::unique_ptr& h_mom_temp, + std::unique_ptr& h_mom_vel, + int flux_normal_axis=0, int flux_direction=0); + +} + +#endif // WARPX_SPECIES_UTILS_H_ diff --git a/Source/Utils/SpeciesUtils.cpp b/Source/Utils/SpeciesUtils.cpp new file mode 100644 index 00000000000..2ac5879f9d1 --- /dev/null +++ b/Source/Utils/SpeciesUtils.cpp @@ -0,0 +1,240 @@ +/* Copyright 2023 Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "SpeciesUtils.H" +#include +#include "Utils/TextMsg.H" +#include "Utils/Parser/ParserUtils.H" + +namespace SpeciesUtils { + + void StringParseAbortMessage(const std::string& var, + const std::string& name) { + std::stringstream stringstream; + std::string string; + stringstream << var << " string '" << name << "' not recognized."; + string = stringstream.str(); + WARPX_ABORT_WITH_MESSAGE(string); + } + + void extractSpeciesProperties (std::string const& species_name, + std::string const& injection_style, amrex::Real& charge, amrex::Real& mass, + PhysicalSpecies& physical_species ) + { + const amrex::ParmParse pp_species_name(species_name); + std::string physical_species_s; + const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); + if (species_is_specified){ + const auto physical_species_from_string = species::from_string( physical_species_s ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, + physical_species_s + " does not exist!"); + physical_species = physical_species_from_string.value(); + charge = species::get_charge( physical_species ); + mass = species::get_mass( physical_species ); + } + + // parse charge and mass + const bool charge_is_specified = + utils::parser::queryWithParser(pp_species_name, "charge", charge); + const bool mass_is_specified = + utils::parser::queryWithParser(pp_species_name, "mass", mass); + + if ( charge_is_specified && species_is_specified ){ + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".charge' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".charge' will take precedence.\n"); + + } + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + charge_is_specified || + species_is_specified || + (injection_style == "external_file"), + "Need to specify at least one of species_type or charge for species '" + + species_name + "'." + ); + + if ( mass_is_specified && species_is_specified ){ + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".mass' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".mass' will take precedence.\n"); + } + } + + // Depending on injection type at runtime, initialize inj_rho + // so that inj_rho->getDensity calls + // InjectorPosition[Constant or Predefined or etc.].getDensity. + void parseDensity (std::string const& species_name, + std::unique_ptr& h_inj_rho, + std::unique_ptr& density_parser) + { + const amrex::ParmParse pp_species_name(species_name); + + // parse density information + std::string rho_prof_s; + pp_species_name.get("profile", rho_prof_s); + std::transform(rho_prof_s.begin(), rho_prof_s.end(), + rho_prof_s.begin(), ::tolower); + if (rho_prof_s == "constant") { + amrex::Real density; + utils::parser::getWithParser(pp_species_name, "density", density); + // Construct InjectorDensity with InjectorDensityConstant. + h_inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); + } else if (rho_prof_s == "predefined") { + // Construct InjectorDensity with InjectorDensityPredefined. + h_inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); + } else if (rho_prof_s == "parse_density_function") { + std::string str_density_function; + utils::parser::Store_parserString( + pp_species_name, "density_function(x,y,z)", str_density_function); + // Construct InjectorDensity with InjectorDensityParser. + density_parser = std::make_unique( + utils::parser::makeParser(str_density_function,{"x","y","z"})); + h_inj_rho.reset(new InjectorDensity((InjectorDensityParser*)nullptr, + density_parser->compile<3>())); + } else { + StringParseAbortMessage("Density profile type", rho_prof_s); + } + } + + // Depending on injection type at runtime, initialize inj_mom + // so that inj_mom->getMomentum calls + // InjectorMomentum[Constant or Gaussian or etc.].getMomentum. + void parseMomentum (std::string const& species_name, const std::string& style, + std::unique_ptr& h_inj_mom, + std::unique_ptr& ux_parser, + std::unique_ptr& uy_parser, + std::unique_ptr& uz_parser, + std::unique_ptr& h_mom_temp, + std::unique_ptr& h_mom_vel, + int flux_normal_axis, int flux_direction) + { + using namespace amrex::literals; + + const amrex::ParmParse pp_species_name(species_name); + + // parse momentum information + std::string mom_dist_s; + pp_species_name.get("momentum_distribution_type", mom_dist_s); + std::transform(mom_dist_s.begin(), + mom_dist_s.end(), + mom_dist_s.begin(), + ::tolower); + if (mom_dist_s == "at_rest") { + constexpr amrex::Real ux = 0._rt; + constexpr amrex::Real uy = 0._rt; + constexpr amrex::Real uz = 0._rt; + // Construct InjectorMomentum with InjectorMomentumConstant. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); + } else if (mom_dist_s == "constant") { + amrex::Real ux = 0._rt; + amrex::Real uy = 0._rt; + amrex::Real uz = 0._rt; + utils::parser::queryWithParser(pp_species_name, "ux", ux); + utils::parser::queryWithParser(pp_species_name, "uy", uy); + utils::parser::queryWithParser(pp_species_name, "uz", uz); + // Construct InjectorMomentum with InjectorMomentumConstant. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); + } else if (mom_dist_s == "gaussian") { + amrex::Real ux_m = 0._rt; + amrex::Real uy_m = 0._rt; + amrex::Real uz_m = 0._rt; + amrex::Real ux_th = 0._rt; + amrex::Real uy_th = 0._rt; + amrex::Real uz_th = 0._rt; + utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); + // Construct InjectorMomentum with InjectorMomentumGaussian. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, + ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); + } else if (mom_dist_s == "gaussianflux") { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(style == "nfluxpercell", + "Error: gaussianflux can only be used with injection_style = NFluxPerCell"); + amrex::Real ux_m = 0._rt; + amrex::Real uy_m = 0._rt; + amrex::Real uz_m = 0._rt; + amrex::Real ux_th = 0._rt; + amrex::Real uy_th = 0._rt; + amrex::Real uz_th = 0._rt; + utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); + // Construct InjectorMomentum with InjectorMomentumGaussianFlux. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianFlux*)nullptr, + ux_m, uy_m, uz_m, ux_th, uy_th, uz_th, + flux_normal_axis, flux_direction)); + } else if (mom_dist_s == "uniform") { + amrex::Real ux_min = 0._rt; + amrex::Real uy_min = 0._rt; + amrex::Real uz_min = 0._rt; + amrex::Real ux_max = 0._rt; + amrex::Real uy_max = 0._rt; + amrex::Real uz_max = 0._rt; + utils::parser::queryWithParser(pp_species_name, "ux_min", ux_min); + utils::parser::queryWithParser(pp_species_name, "uy_min", uy_min); + utils::parser::queryWithParser(pp_species_name, "uz_min", uz_min); + utils::parser::queryWithParser(pp_species_name, "ux_max", ux_max); + utils::parser::queryWithParser(pp_species_name, "uy_max", uy_max); + utils::parser::queryWithParser(pp_species_name, "uz_max", uz_max); + // Construct InjectorMomentum with InjectorMomentumUniform. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumUniform*)nullptr, + ux_min, uy_min, uz_min, ux_max, uy_max, uz_max)); + } else if (mom_dist_s == "maxwell_boltzmann"){ + h_mom_temp = std::make_unique(pp_species_name); + const GetTemperature getTemp(*h_mom_temp); + h_mom_vel = std::make_unique(pp_species_name); + const GetVelocity getVel(*h_mom_vel); + // Construct InjectorMomentum with InjectorMomentumBoltzmann. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, getTemp, getVel)); + } else if (mom_dist_s == "maxwell_juttner"){ + h_mom_temp = std::make_unique(pp_species_name); + const GetTemperature getTemp(*h_mom_temp); + h_mom_vel = std::make_unique(pp_species_name); + const GetVelocity getVel(*h_mom_vel); + // Construct InjectorMomentum with InjectorMomentumJuttner. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, getTemp, getVel)); + } else if (mom_dist_s == "radial_expansion") { + amrex::Real u_over_r = 0._rt; + utils::parser::queryWithParser(pp_species_name, "u_over_r", u_over_r); + // Construct InjectorMomentum with InjectorMomentumRadialExpansion. + h_inj_mom.reset(new InjectorMomentum + ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); + } else if (mom_dist_s == "parse_momentum_function") { + std::string str_momentum_function_ux; + std::string str_momentum_function_uy; + std::string str_momentum_function_uz; + utils::parser::Store_parserString(pp_species_name, "momentum_function_ux(x,y,z)", + str_momentum_function_ux); + utils::parser::Store_parserString(pp_species_name, "momentum_function_uy(x,y,z)", + str_momentum_function_uy); + utils::parser::Store_parserString(pp_species_name, "momentum_function_uz(x,y,z)", + str_momentum_function_uz); + // Construct InjectorMomentum with InjectorMomentumParser. + ux_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_ux, {"x","y","z"})); + uy_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uy, {"x","y","z"})); + uz_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uz, {"x","y","z"})); + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumParser*)nullptr, + ux_parser->compile<3>(), + uy_parser->compile<3>(), + uz_parser->compile<3>())); + } else { + StringParseAbortMessage("Momentum distribution type", mom_dist_s); + } + } + +} diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index f968af81c13..62fabef2542 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -13,6 +13,8 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" @@ -357,6 +359,18 @@ WarpX::MoveWindow (const int step, bool move_j) } } } + + // Shift values of N, NU for each fluid species + if (do_fluid_species) { + const int n_fluid_species = myfl->nSpecies(); + for (int i=0; iGetFluidContainer(i); + shiftMF( *fl.N[lev], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *fl.NU[lev][0], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *fl.NU[lev][1], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *fl.NU[lev][2], geom[lev], num_shift, dir, lev, do_update_cost ); + } + } } // Loop over species (particles and lasers) @@ -410,6 +424,30 @@ WarpX::MoveWindow (const int step, bool move_j) } } + // Continuously inject fluid species in new cells (by default only on level 0) + const int lev = 0; + // Find box in which to initialize new fluid cells + amrex::Box injection_box = geom[lev].Domain(); + injection_box.surroundingNodes(); // get nodal box + // Restrict box in the direction of the moving window, to only include the new cells + if (moving_window_v > 0._rt) + { + injection_box.setSmall( dir, injection_box.bigEnd(dir) - num_shift_base + 1 ); + } + else if (moving_window_v < 0._rt) + { + injection_box.setBig( dir, injection_box.smallEnd(dir) + num_shift_base - 1 ); + } + // Loop over fluid species, and fill the values of the new cells + if (do_fluid_species) { + const int n_fluid_species = myfl->nSpecies(); + const amrex::Real cur_time = t_new[0]; + for (int i=0; iGetFluidContainer(i); + fl.InitData( lev, injection_box, cur_time ); + } + } + return num_shift_base; } diff --git a/Source/WarpX.H b/Source/WarpX.H index 051acb80b56..0e581c4593a 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -23,6 +23,9 @@ #include "Particles/ParticleBoundaryBuffer_fwd.H" #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer_fwd.H" +#include "Fluids/MultiFluidContainer_fwd.H" +#include "Fluids/WarpXFluidContainer_fwd.H" + #ifdef WARPX_USE_PSATD # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ_fwd.H" @@ -117,6 +120,7 @@ public: void Evolve (int numsteps = -1); MultiParticleContainer& GetPartContainer () { return *mypc; } + MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} @@ -507,6 +511,7 @@ public: const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} bool DoPML () const {return do_pml;} + bool DoFluidSpecies () const {return do_fluid_species;} #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) const PML_RZ* getPMLRZ() {return pml_rz[0].get();} @@ -1336,6 +1341,10 @@ private: std::unique_ptr mypc; std::unique_ptr multi_diags; + // Fluid container + bool do_fluid_species = 0; + std::unique_ptr myfl; + // // Fields: First array for level, second for direction // diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4e3269b4156..f60d52586b8 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -30,6 +30,8 @@ #include "FieldSolver/WarpX_FDTD.H" #include "Filter/NCIGodfreyFilter.H" #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Particles/ParticleBoundaryBuffer.H" #include "AcceleratorLattice/AcceleratorLattice.H" #include "Utils/TextMsg.H" @@ -318,6 +320,11 @@ WarpX::WarpX () // Particle Boundary Buffer (i.e., scraped particles on boundary) m_particle_boundary_buffer = std::make_unique(); + // Fluid Container + if (do_fluid_species) { + myfl = std::make_unique(nlevs_max); + } + Efield_aux.resize(nlevs_max); Bfield_aux.resize(nlevs_max); @@ -873,26 +880,30 @@ WarpX::ReadParameters () quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); } - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !( - ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML && - WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller ) || - ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML && - WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller ) - ), - "PML and Silver-Mueller boundary conditions cannot be activated at the same time."); + const auto at_least_one_boundary_is_pml = + (std::any_of(WarpX::field_boundary_lo.begin(), WarpX::field_boundary_lo.end(), + [](const auto& cc){return cc == FieldBoundaryType::PML;}) + || + std::any_of(WarpX::field_boundary_hi.begin(), WarpX::field_boundary_hi.end(), + [](const auto& cc){return cc == FieldBoundaryType::PML;}) + ); + const auto at_least_one_boundary_is_silver_mueller = + (std::any_of(WarpX::field_boundary_lo.begin(), WarpX::field_boundary_lo.end(), + [](const auto& cc){return cc == FieldBoundaryType::Absorbing_SilverMueller;}) + || + std::any_of(WarpX::field_boundary_hi.begin(), WarpX::field_boundary_hi.end(), + [](const auto& cc){return cc == FieldBoundaryType::Absorbing_SilverMueller;}) + ); - if (WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller || - WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller) - { - // SilverMueller is implemented for Yee - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee, - "The Silver-Mueller boundary condition can only be used with the Yee solver."); - } - } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !(at_least_one_boundary_is_pml && at_least_one_boundary_is_silver_mueller), + "PML and Silver-Mueller boundary conditions cannot be activated at the same time."); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (!at_least_one_boundary_is_silver_mueller) || + (electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee), + "The Silver-Mueller boundary condition can only be used with the Yee solver."); utils::parser::queryWithParser(pp_warpx, "pml_ncell", pml_ncell); utils::parser::queryWithParser(pp_warpx, "pml_delta", pml_delta); @@ -1017,6 +1028,25 @@ WarpX::ReadParameters () "The number of azimuthal modes (n_rz_azimuthal_modes) must be at least 1"); #endif + // Check whether fluid species will be used + { + const ParmParse pp_fluids("fluids"); + std::vector fluid_species_names = {}; + pp_fluids.queryarr("species_names", fluid_species_names); + if (!fluid_species_names.empty()) do_fluid_species = 1; + if (do_fluid_species) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level <= 1, + "Fluid species cannot currently be used with mesh refinement."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electrostatic_solver_id != ElectrostaticSolverAlgo::Relativistic, + "Fluid species cannot currently be used with the relativistic electrostatic solver."); +#ifdef WARPX_DIM_RZ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( n_rz_azimuthal_modes <= 1, + "Fluid species cannot be used with more than 1 azimuthal mode."); +#endif + } + } + // Set default parameters with hybrid grid (parsed later below) if (grid_type == GridType::Hybrid) { @@ -2232,6 +2262,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm ); } + // Allocate extra multifabs needed for fluids + if (do_fluid_species) { + myfl->AllocateLevelMFs(lev, ba, dm); + auto & warpx = GetInstance(); + const amrex::Real cur_time = warpx.gett_new(lev); + myfl->InitData(lev, geom[lev].Domain(),cur_time); + } + if (fft_do_time_averaging) { AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]"); diff --git a/Tools/Algorithms/stencil.py b/Tools/Algorithms/stencil.py index cb24a62148b..63bb7f11c2e 100644 --- a/Tools/Algorithms/stencil.py +++ b/Tools/Algorithms/stencil.py @@ -13,7 +13,9 @@ import argparse import os +import sys +sys.path.append('../Parser/') from input_file_parser import parse_input_file import matplotlib.pyplot as plt import numpy as np diff --git a/Tools/Parser/__init__.py b/Tools/Parser/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Tools/Algorithms/input_file_parser.py b/Tools/Parser/input_file_parser.py similarity index 100% rename from Tools/Algorithms/input_file_parser.py rename to Tools/Parser/input_file_parser.py diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh new file mode 100755 index 00000000000..223b41d1967 --- /dev/null +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was lassen_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SRC_DIR="/usr/workspace/${USER}/lassen/src" +SW_DIR="/usr/workspace/${USER}/lassen/gpu" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10 + +# HDF5 +if [ -d ${SRC_DIR}/hdf5 ] +then + cd ${SRC_DIR}/hdf5 + git fetch --prune + git checkout hdf5-1_14_1-2 + cd - +else + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 +fi +cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 +cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10 + +# ADIOS2 +if [ -d ${SRC_DIR}/adios2 ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-lassen-build --target install -j 10 + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ] +then + cd ${SRC_DIR}/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ] +then + cd ${SRC_DIR}/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 + + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/warpx-lassen +python3 -m venv ${SW_DIR}/venvs/warpx-lassen +source ${SW_DIR}/venvs/warpx-lassen/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade -Ccompile-args="-j10" scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself +echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt +python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# for ML dependencies, see install_v100_ml.sh + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/lassen-llnl/install_v100_ml.sh b/Tools/machines/lassen-llnl/install_v100_ml.sh new file mode 100755 index 00000000000..6e00be035d6 --- /dev/null +++ b/Tools/machines/lassen-llnl/install_v100_ml.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was lassen_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SRC_DIR="/usr/workspace/${USER}/lassen/src" +mkdir -p ${SRC_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qqq -y torch 2>/dev/null || true + + +# Python ML ################################################################### +# +# for basic python dependencies, see install_v100_dependencies.sh + +# optional: for libEnsemble - WIP: issues with nlopt +# python3 -m pip install -r ${SRC_DIR}/warpx/Tools/LibEnsemble/requirements.txt + +# optional: for pytorch +if [ -d ${SRC_DIR}/pytorch ] +then + cd ${SRC_DIR}/pytorch + git fetch + git checkout . + git checkout v2.0.1 + git submodule update --init --recursive + cd - +else + git clone -b v2.0.1 --recurse-submodules https://github.com/pytorch/pytorch.git ${SRC_DIR}/pytorch +fi +cd ${SRC_DIR}/pytorch +rm -rf build + +# see https://github.com/pytorch/pytorch/issues/97497#issuecomment-1499069641 +# https://github.com/pytorch/pytorch/pull/98511 +wget -q -O - https://github.com/pytorch/pytorch/pull/98511.patch | git apply + +python3 -m pip install -r requirements.txt + +# see https://github.com/pytorch/pytorch/issues/108984#issuecomment-1712938737 +LDFLAGS="-L${CUDA_HOME}/nvidia/targets/ppc64le-linux/lib/" \ +USE_CUDA=1 BLAS=OpenBLAS MAX_JOBS=64 ATEN_AVX512_256=OFF BUILD_TEST=0 python3 setup.py develop +# (optional) If using torch.compile with inductor/triton, install the matching version of triton +#make triton +rm -rf build +cd - + +# optional: optimas dependencies (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# commented because scikit-learn et al. compile > 2 hrs +# please run manually on a login node if needed +#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/lassen-llnl/lassen.bsub b/Tools/machines/lassen-llnl/lassen_v100.bsub similarity index 100% rename from Tools/machines/lassen-llnl/lassen.bsub rename to Tools/machines/lassen-llnl/lassen_v100.bsub diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example new file mode 100644 index 00000000000..652af2a2822 --- /dev/null +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example @@ -0,0 +1,54 @@ +# please set your project account +#export proj="" # edit this and comment in + +# required dependencies +module load cmake/3.23.1 +module load gcc/11.2.1 +module load cuda/12.0.0 + +# optional: for QED lookup table generation support +module load boost/1.70.0 + +# optional: for openPMD support +SRC_DIR="/usr/workspace/${USER}/lassen/src" +SW_DIR="/usr/workspace/${USER}/lassen/gpu" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH + +# optional: for Python bindings +module load python/3.8.2 + +if [ -d "${SW_DIR}/venvs/warpx-lassen" ] +then + source ${SW_DIR}/venvs/warpx-lassen/bin/activate +fi + +# optional: an alias to request an interactive node for two hours +alias getNode="bsub -G $proj -W 2:00 -nnodes 1 -Is /bin/bash" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" + +# fix system defaults: do not escape $ with a \ on tab completion +shopt -s direxpand + +# optimize CUDA compilation for V100 +export AMREX_CUDA_ARCH=7.0 +export CUDAARCHS=70 + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lassen-llnl/lassen_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_warpx.profile.example deleted file mode 100644 index fb605b53ec2..00000000000 --- a/Tools/machines/lassen-llnl/lassen_warpx.profile.example +++ /dev/null @@ -1,44 +0,0 @@ -# please set your project account -#export proj="" # edit this and comment in - -# required dependencies -module load cmake/3.23.1 -module load clang/12.0.1-gcc-8.3.1 -module load cuda/12.0.0 - -# optional: for QED lookup table generation support -module load boost/1.70.0 - -# optional: for openPMD support -export CMAKE_PREFIX_PATH=$HOME/sw/lassen/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lassen/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lassen/adios2-2.8.3:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=$HOME/sw/lassen/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lassen/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lassen/adios2-2.8.3/lib64:$LD_LIBRARY_PATH - -# optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=$HOME/sw/lassen/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lassen/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=$HOME/sw/lassen/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lassen/lapackpp-master/lib64:$LD_LIBRARY_PATH - -# optional: for Python bindings -module load python/3.8.2 - -# optional: an alias to request an interactive node for two hours -alias getNode="bsub -G $proj -W 2:00 -nnodes 1 -Is /bin/bash" - -# fix system defaults: do not escape $ with a \ on tab completion -shopt -s direxpand - -# optimize CUDA compilation for V100 -export AMREX_CUDA_ARCH=7.0 -export CUDAARCHS=70 - -# compiler environment hints -export CC=$(which clang) -export CXX=$(which clang++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=$(which clang++) diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index a60c3ba7594..7bcc7053974 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -127,6 +127,7 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install cupy-cuda11x # CUDA 11.7 compatible wheel # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index ab5a0c963b8..d25b4825dde 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -11,10 +11,10 @@ module load cmake/3.22.0 module load cray-fftw/3.3.10.3 # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-22.11/83104/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.80.0-ute7nbx4wmfrw53q7hyh6wyezub5ljry +export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.1 +module load cray-hdf5-parallel/1.12.2.7 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -26,7 +26,7 @@ export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/li export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-22.05/78535/spack/opt/spack/cray-sles15-zen3/gcc-11.2.0/ccache-4.5.1-ybl7xefvggn6hov4dsdxxnztji74tolj/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH # optional: for Python bindings or libEnsemble module load cray-python/3.9.13.1 diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 09d6a61b458..af3e69c1190 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -9,10 +9,10 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo module load cmake/3.22.0 # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-22.11/83104/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.80.0-ute7nbx4wmfrw53q7hyh6wyezub5ljry +export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.1 +module load cray-hdf5-parallel/1.12.2.7 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -24,7 +24,7 @@ export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-22.05/78535/spack/opt/spack/cray-sles15-zen3/gcc-11.2.0/ccache-4.5.1-ybl7xefvggn6hov4dsdxxnztji74tolj/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH # optional: for Python bindings or libEnsemble module load cray-python/3.9.13.1 diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh new file mode 100755 index 00000000000..c162ac5d390 --- /dev/null +++ b/Tools/machines/quartz-llnl/install_dependencies.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was quartz_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your quartz_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SW_DIR="/usr/workspace/${USER}/quartz" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with ${HOME} and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d ${HOME}/src/c-blosc ] +then + cd ${HOME}/src/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${HOME}/src/c-blosc +fi +cmake -S ${HOME}/src/c-blosc -B ${build_dir}/c-blosc-quartz-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-quartz-build --target install --parallel 6 + +# ADIOS2 +if [ -d ${HOME}/src/adios2 ] +then + cd ${HOME}/src/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${HOME}/src/adios2 +fi +cmake -S ${HOME}/src/adios2 -B ${build_dir}/adios2-quartz-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-quartz-build --target install -j 6 + +# BLAS++ (for PSATD+RZ) +if [ -d ${HOME}/src/blaspp ] +then + cd ${HOME}/src/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp +fi +cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-quartz-build --target install --parallel 6 + +# LAPACK++ (for PSATD+RZ) +if [ -d ${HOME}/src/lapackpp ] +then + cd ${HOME}/src/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp +fi +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-quartz-build --target install --parallel 6 + + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/warpx-quartz +python3 -m venv ${SW_DIR}/venvs/warpx-quartz +source ${SW_DIR}/venvs/warpx-quartz/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${HOME}/src/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch + + +# remove build temporary directory ############################################ +# +rm -rf ${build_dir} diff --git a/Tools/machines/quartz-llnl/quartz_warpx.profile.example b/Tools/machines/quartz-llnl/quartz_warpx.profile.example index 370e4a601ac..810005bafb7 100644 --- a/Tools/machines/quartz-llnl/quartz_warpx.profile.example +++ b/Tools/machines/quartz-llnl/quartz_warpx.profile.example @@ -1,37 +1,53 @@ # please set your project account -#export proj= +#export proj="" # edit this and comment in # required dependencies -module load cmake/3.20.2 -module load intel/2021.4 -module load mvapich2/2.3 +module load cmake/3.23.1 +module load clang/14.0.6-magic +module load mvapich2/2.3.7 # optional: for PSATD support -module load fftw/3.3.8 +module load fftw/3.3.10 # optional: for QED lookup table generation support -module load boost/1.73.0 +module load boost/1.80.0 # optional: for openPMD support -# TODO ADIOS2 -module load hdf5-parallel/1.10.2 +module load hdf5-parallel/1.14.0 + +SW_DIR="/usr/workspace/${USER}/quartz" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH # optional: for PSATD in RZ geometry support -# TODO: blaspp lapackpp +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for Python bindings -module load python/3.8.2 +module load python/3.9.12 + +if [ -d "${SW_DIR}/venvs/warpx-quartz" ] +then + source ${SW_DIR}/venvs/warpx-quartz/bin/activate +fi # optional: an alias to request an interactive node for two hours alias getNode="srun --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=18 -p pdebug --pty bash" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=18 -p pdebug" # fix system defaults: do not escape $ with a \ on tab completion shopt -s direxpand +# optimize CPU microarchitecture for Intel Xeon E5-2695 v4 +# note: the cc/CC/ftn wrappers below add those +export CXXFLAGS="-march=broadwell" +export CFLAGS="-march=broadwell" + # compiler environment hints -export CC=$(which icc) -export CXX=$(which icpc) -export FC=$(which ifort) -# we need a newer libstdc++: -export CFLAGS="-gcc-name=/usr/tce/packages/gcc/gcc-8.3.1/bin/gcc ${CFLAGS}" -export CXXFLAGS="-gxx-name=/usr/tce/packages/gcc/gcc-8.3.1/bin/g++ ${CXXFLAGS}" +export CC=$(which clang) +export CXX=$(which clang++) +export FC=$(which gfortran) diff --git a/Tools/machines/summit-olcf/install_gpu_ml.sh b/Tools/machines/summit-olcf/install_gpu_ml.sh index ff4eaf1555c..25a8eb27abd 100755 --- a/Tools/machines/summit-olcf/install_gpu_ml.sh +++ b/Tools/machines/summit-olcf/install_gpu_ml.sh @@ -60,6 +60,7 @@ then git fetch git checkout . git checkout v2.0.1 + git submodule update --init --recursive cd - else git clone -b v2.0.1 --recurse-submodules https://github.com/pytorch/pytorch.git /ccs/proj/${proj}/${USER}/src/pytorch diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index c060102d6e8..5d88bb9aeed 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -63,7 +63,7 @@ fi # for paralle execution, start on the batch node: jsrun alias getNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -Is /bin/bash" # an alias to run a command on a batch node for up to 30min -# usage: nrun +# usage: runNode alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" # fix system defaults: do not escape $ with a \ on tab completion diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 1e2697a4e50..3e0f044f020 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -257,7 +257,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "23.09" +set(WarpX_amrex_branch "2e99628138df3b5b0ecf50b0c1201d5547f821a0" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake index d04269b8876..8400dc06a4b 100644 --- a/cmake/dependencies/PICSAR.cmake +++ b/cmake/dependencies/PICSAR.cmake @@ -106,7 +106,7 @@ if(WarpX_QED) set(WarpX_picsar_repo "https://github.com/ECP-WarpX/picsar.git" CACHE STRING "Repository URI to pull and build PICSAR from if(WarpX_picsar_internal)") - set(WarpX_picsar_branch "aa54e985398c1d575abc7e6737cdbc660a13765f" + set(WarpX_picsar_branch "23.09" CACHE STRING "Repository branch for WarpX_picsar_repo if(WarpX_picsar_internal)") diff --git a/run_test.sh b/run_test.sh index faf4bdd85c4..81237fb6971 100755 --- a/run_test.sh +++ b/run_test.sh @@ -71,7 +71,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 23.09 && cd - +cd amrex && git checkout --detach 2e99628138df3b5b0ecf50b0c1201d5547f821a0 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets