Merge branch 'development' into fix_magentostatic_no_eb

.github/workflows/cuda.yml

@@ -131,7 +131,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 028638564f7be0694b9898f8d4088cdbf9a6f9f5 && cd -
+        cd ../amrex && git checkout --detach 3734079379bb6b2a3850d197241f6b2c3b3bfa7d && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

.github/workflows/windows.yml

@@ -10,7 +10,9 @@ jobs:
   build_win_msvc:
     name: MSVC C++17 w/o MPI
     runs-on: windows-latest
-    if: github.event.pull_request.draft == false
+    # disabled due to issues in #5230
+    if: 0
+    #if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5

.pre-commit-config.yaml

@@ -69,7 +69,7 @@ repos:
 # Python: Ruff linter & formatter
 #   https://docs.astral.sh/ruff/
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.5
+  rev: v0.6.7
   hooks:
     # Run the linter
     - id: ruff

Docs/source/highlights.rst

@@ -90,6 +90,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction.
    Physical Review Research **6**, 033148, 2024.
    `DOI:10.1103/PhysRevResearch.6.033148 <https://doi.org/10.1103/PhysRevResearch.6.033148>`__
 
+#. Zaïm N, Sainte-Marie A, Fedeli L, Bartoli P, Huebl A, Leblanc A, Vay J-L, Vincenti H.
+   **Light-matter interaction near the Schwinger limit using tightly focused doppler-boosted lasers**.
+   Physical Review Letters **132**, 175002, 2024.
+   `DOI:10.1103/PhysRevLett.132.175002 <https://doi.org/10.1103/PhysRevLett.132.175002>`__
+
 #. Knight B, Gautam C, Stoner C, Egner B, Smith J, Orban C, Manfredi J, Frische K, Dexter M, Chowdhury E, Patnaik A (2023).
    **Detailed Characterization of a kHz-rate Laser-Driven Fusion at a Thin Liquid Sheet with a Neutron Detection Suite**.
    High Power Laser Science and Engineering, 1-13, 2023.
@@ -110,6 +115,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction.
    Phys. Rev. Accel. Beams **25**, 093402, 2022.
    `DOI:10.1103/PhysRevAccelBeams.25.093402 <https://doi.org/10.1103/PhysRevAccelBeams.25.093402>`__
 
+#. Fedeli L, Sainte-Marie A, Zaïm N, Thévenet M, Vay J-L, Myers A, Quéré F, Vincenti H.
+   **Probing strong-field QED with Doppler-boosted PetaWatt-class lasers**.
+   Physical Review Letters **127**, 114801, 2021.
+   `DOI:10.1103/PhysRevLett.127.114801 <https://doi.org/10.1103/PhysRevLett.127.114801>`__
+
 
 Particle Accelerator & Beam Physics
 ***********************************

Docs/source/install/hpc.rst

@@ -43,6 +43,7 @@ This section documents quick-start guides for a selection of supercomputers that
    hpc/lassen
    hpc/lawrencium
    hpc/leonardo
+   hpc/lonestar6
    hpc/lumi
    hpc/lxplus
    hpc/ookami

Docs/source/install/hpc/lonestar6.rst

@@ -0,0 +1,139 @@
+.. _building-lonestar6:
+
+Lonestar6 (TACC)
+================
+
+The `Lonestar6 cluster <https://portal.tacc.utexas.edu/user-guides/lonestar6>`_ is located at `TACC <https://www.tacc.utexas.edu>`__.
+
+
+Introduction
+------------
+
+If you are new to this system, **please see the following resources**:
+
+* `TACC user guide <https://portal.tacc.utexas.edu/user-guides/>`__
+* Batch system: `Slurm <https://portal.tacc.utexas.edu/user-guides/lonestar6#job-management>`__
+* `Jupyter service <https://tacc.github.io/ctls2017/docs/intro_to_python/intro_to_python_011_jupyter.html>`__
+* `Filesystem directories <https://portal.tacc.utexas.edu/user-guides/lonestar6#managing-files-on-lonestar6>`__:
+
+  * ``$HOME``: per-user home directory, backed up (10 GB)
+  * ``$WORK``: per-user production directory, not backed up, not purged, Lustre (1 TB)
+  * ``$SCRATCH``: per-user production directory, not backed up, purged every 10 days, Lustre (no limits, 8PByte total)
+
+
+Installation
+------------
+
+Use the following commands to download the WarpX source code and switch to the correct branch:
+
+.. code-block:: bash
+
+   git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx
+
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/lonestar6_warpx_a100.profile``.
+Create it now:
+
+.. code-block:: bash
+
+   cp $HOME/src/warpx/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example $HOME/lonestar6_warpx_a100.profile
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example
+      :language: bash
+
+Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+For example, if you are member of the project ``abcde``, then run ``nano $HOME/lonestar6_warpx_a100.profile`` and edit line 2 to read:
+
+.. code-block:: bash
+
+   export proj="abcde"
+
+Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit).
+
+.. important::
+
+   Now, and as the first step on future logins to Lonestar6, activate these environment settings:
+
+   .. code-block:: bash
+
+      source $HOME/lonestar6_warpx_a100.profile
+
+Finally, since Lonestar6 does not yet provide software modules for some of our dependencies, install them once:
+
+.. code-block:: bash
+
+   bash $HOME/src/warpx/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh
+   source ${SW_DIR}/venvs/warpx-a100/bin/activate
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/install_a100_dependencies.sh
+      :language: bash
+
+
+.. _building-lonestar6-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands <building-cmake>` to compile the application executable:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+   rm -rf build_pm_gpu
+
+   cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_gpu -j 16
+
+The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``.
+Additionally, the following commands will install WarpX as a Python module:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+   rm -rf build_pm_gpu_py
+
+   cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_gpu_py -j 16 --target pip_install
+
+Now, you can :ref:`submit Lonestar6 compute jobs <running-cpp-lonestar6>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
+Or, you can use the WarpX executables to submit Lonestar6 jobs (:ref:`example inputs <usage-examples>`).
+For executables, you can reference their location in your :ref:`job script <running-cpp-lonestar6>` or copy them to a location in ``$WORK`` or ``$SCRATCH``.
+
+
+.. _running-cpp-lonestar6:
+
+Running
+-------
+
+.. _running-cpp-lonestar6-A100-GPUs:
+
+A100 GPUs (40 GB)
+^^^^^^^^^^^^^^^^^
+
+`84 GPU nodes, each with 2 A100 GPUs (40 GB) <https://portal.tacc.utexas.edu/user-guides/lonestar6#system-gpu>`__.
+
+The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly) on the supercomputer lonestar6 at tacc.
+Replace descriptions between chevrons ``<>`` by relevant values, for instance ``<input file>`` could be ``plasma_mirror_inputs``.
+Note that we run one MPI rank per GPU.
+
+
+.. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch
+   :language: bash
+   :caption: You can copy this file from ``Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch``.
+
+To run a simulation, copy the lines above to a file ``lonestar6.sbatch`` and run
+
+.. code-block:: bash
+
+   sbatch lonestar6_a100.sbatch
+
+to submit the job.

Docs/source/usage/parameters.rst

@@ -2129,14 +2129,24 @@ Time step
     The ratio between the actual timestep that is used in the simulation
     and the Courant-Friedrichs-Lewy (CFL) limit. (e.g. for `warpx.cfl=1`,
     the timestep will be exactly equal to the CFL limit.)
-    This parameter will only be used with the electromagnetic solver.
+    For some speed v and grid spacing dx, this limits the timestep to `warpx.cfl * dx / v`.
+    When used with the electromagnetic solver, `v` is the speed of light.
+    For the electrostatic solver, `v` is the maximum speed among all particles in the domain.
 
 * ``warpx.const_dt`` (`float`)
     Allows direct specification of the time step size, in units of seconds.
-    When the electrostatic solver is being used, this must be supplied.
+    When the electrostatic solver is being used, this must be supplied if not using adaptive timestepping.
     This can be used with the electromagnetic solver, overriding ``warpx.cfl``, but
     it is up to the user to ensure that the CFL condition is met.
 
+* ``warpx.dt_update_interval`` (`string`) optional (default `-1`)
+    How many iterations pass between timestep adaptations when using the electrostatic solver.
+    Must be greater than `0` to use adaptive timestepping, or else ``warpx.const_dt`` must be specified.
+
+* ``warpx.max_dt`` (`float`) optional
+    The maximum timestep permitted for the electrostatic solver, when using adaptive timestepping.
+    If supplied, also sets the initial timestep for these simulations, before the first timestep update.
+
 Filtering
 ^^^^^^^^^
 
@@ -3448,39 +3458,42 @@ Reduced Diagnostics
         For 1D-Z, :math:`x`-related and :math:`y`-related quantities are not outputted.
         RZ geometry is not supported yet.
 
-* ``DifferentialLuminosity``
-    This type computes the differential luminosity between two species, defined as:
+    * ``DifferentialLuminosity``
+        This type computes the differential luminosity between two species, defined as:
 
-    .. math::
+        .. math::
+
+            \frac{d\mathcal{L}}{d\mathcal{E}^*}(\mathcal{E}^*, t) = \int_0^t dt'\int d\boldsymbol{x}\,d\boldsymbol{p}_1 d\boldsymbol{p}_2\;
+             \sqrt{ |\boldsymbol{v}_1 - \boldsymbol{v}_2|^2 - |\boldsymbol{v}_1\times\boldsymbol{v}_2|^2/c^2} \\ f_1(\boldsymbol{x}, \boldsymbol{p}_1, t')f_2(\boldsymbol{x}, \boldsymbol{p}_2, t') \delta(\mathcal{E}^* - \mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2))
 
-        \frac{d\mathcal{L}}{d\mathcal{E}^*}(\mathcal{E}^*, t) = \int_0^t dt'\int d\boldsymbol{x}\,d\boldsymbol{p}_1 d\boldsymbol{p}_2\;
-         \sqrt{ |\boldsymbol{v}_1 - \boldsymbol{v}_2|^2 - |\boldsymbol{v}_1\times\boldsymbol{v}_2|^2/c^2} \\ f_1(\boldsymbol{x}, \boldsymbol{p}_1, t')f_2(\boldsymbol{x}, \boldsymbol{p}_2, t') \delta(\mathcal{E}^* - \mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2))
+        where :math:`\mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2) = \sqrt{m_1^2c^4 + m_2^2c^4 + 2(m_1 m_2 c^4
+        \gamma_1 \gamma_2 - \boldsymbol{p}_1\cdot\boldsymbol{p}_2 c^2)}` is the energy in the center-of-mass frame,
+        and :math:`f_i` is the distribution function of species :math:`i`. Note that, if :math:`\sigma^*(\mathcal{E}^*)`
+        is the center-of-mass cross-section of a given collision process, then
+        :math:`\int d\mathcal{E}^* \frac{d\mathcal{L}}{d\mathcal{E}^*} (\mathcal{E}^*, t)\sigma^*(\mathcal{E}^*)`
+        gives the total number of collisions of that process (from the beginning of the simulation up until time :math:`t`).
 
-    where :math:`\mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2) = \sqrt{m_1^2c^4 + m_2^2c^4 + 2(m_1 m_2 c^4
-    \gamma_1 \gamma_2 - \boldsymbol{p}_1\cdot\boldsymbol{p}_2 c^2)}` is the energy in the center-of-mass frame,
-    and :math:`f_i` is the distribution function of species :math:`i`. Note that, if :math:`\sigma^*(\mathcal{E}^*)`
-    is the center-of-mass cross-section of a given collision process, then
-    :math:`\int d\mathcal{E}^* \frac{d\mathcal{L}}{d\mathcal{E}^*} (\mathcal{E}^*, t)\sigma^*(\mathcal{E}^*)`
-    gives the total number of collisions of that process (from the beginning of the simulation up until time :math:`t`).
+         The differential luminosity is given in units of :math:`\text{m}^{-2}.\text{eV}^{-1}`. For collider-relevant WarpX simulations
+        involving two crossing, high-energy beams of particles, the differential luminosity in :math:`\text{s}^{-1}.\text{m}^{-2}.\text{eV}^{-1}`
+        can be obtained by multiplying the above differential luminosity by the expected repetition rate of the beams.
 
-    The differential luminosity is given in units of :math:`\text{m}^{-2}.\text{eV}^{-1}`. For collider-relevant WarpX simulations
-    involving two crossing, high-energy beams of particles, the differential luminosity in :math:`\text{s}^{-1}.\text{m}^{-2}.\text{eV}^{-1}`
-    can be obtained by multiplying the above differential luminosity by the expected repetition rate of the beams.
+        In practice, the above expression of the differential luminosity is evaluated over discrete bins in energy :math:`\mathcal{E}^*`,
+        and by summing over macroparticles.
 
-    In practice, the above expression of the differential luminosity is evaluated over discrete bins in energy :math:`\mathcal{E}^*`,
-    and by summing over macroparticles.
+        * ``<reduced_diags_name>.species`` (`list of two strings`)
+            The names of the two species for which the differential luminosity is computed.
 
-    * ``<reduced_diags_name>.species`` (`list of two strings`)
-        The names of the two species for which the differential luminosity is computed.
+        * ``<reduced_diags_name>.bin_number`` (`int` > 0)
+            The number of bins in energy :math:`\mathcal{E}^*`
 
-    * ``<reduced_diags_name>.bin_number`` (`int` > 0)
-        The number of bins in energy :math:`\mathcal{E}^*`
+        * ``<reduced_diags_name>.bin_max`` (`float`, in eV)
+            The minimum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed.
 
-    * ``<reduced_diags_name>.bin_max`` (`float`, in eV)
-        The minimum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed.
+        * ``<reduced_diags_name>.bin_min`` (`float`, in eV)
+            The maximum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed.
 
-    * ``<reduced_diags_name>.bin_min`` (`float`, in eV)
-        The maximum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed.
+    * ``Timestep``
+        This type outputs the simulation's physical timestep (in seconds) at each mesh refinement level.
 
 * ``<reduced_diags_name>.intervals`` (`string`)
     Using the `Intervals Parser`_ syntax, this string defines the timesteps at which reduced

Examples/Tests/electrostatic_sphere/CMakeLists.txt

@@ -41,6 +41,16 @@ add_warpx_test(
     OFF  # dependency
 )
 
+add_warpx_test(
+    test_3d_electrostatic_sphere_adaptive  # name
+    3  # dims
+    2  # nprocs
+    inputs_test_3d_electrostatic_sphere_adaptive  # inputs
+    analysis_electrostatic_sphere.py  # analysis
+    diags/diag1000054  # output
+    OFF  # dependency
+)
+
 add_warpx_test(
     test_rz_electrostatic_sphere  # name
     RZ  # dims

Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive

@@ -0,0 +1,47 @@
+stop_time = 60e-6
+warpx.cfl = 0.2
+warpx.dt_update_interval = 10
+warpx.max_dt = 1.5e-6
+amr.n_cell = 64 64 64
+amr.max_level = 0
+amr.blocking_factor = 8
+amr.max_grid_size = 128
+geometry.dims = 3
+geometry.prob_lo = -0.5 -0.5 -0.5
+geometry.prob_hi =  0.5  0.5  0.5
+boundary.field_lo = pec pec pec
+boundary.field_hi = pec pec pec
+warpx.do_electrostatic = relativistic
+
+particles.species_names = electron
+
+algo.field_gathering = momentum-conserving
+
+# Order of particle shape factors
+algo.particle_shape = 1
+
+my_constants.n0 = 1.49e6
+my_constants.R0 = 0.1
+
+electron.charge = -q_e
+electron.mass = m_e
+electron.injection_style = "NUniformPerCell"
+electron.num_particles_per_cell_each_dim = 2 2 2
+electron.profile = parse_density_function
+electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0"
+electron.momentum_distribution_type = at_rest
+
+diagnostics.diags_names = diag1 diag2
+
+diag1.intervals = 30
+diag1.diag_type = Full
+diag1.fields_to_plot = Ex Ey Ez rho
+
+diag2.intervals = 30
+diag2.diag_type = Full
+diag2.fields_to_plot = none
+diag2.format = openpmd
+
+warpx.reduced_diags_names = timestep
+timestep.intervals = 1
+timestep.type = Timestep

Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py

@@ -94,7 +94,7 @@
 face_areas_y = fields.FaceAreasyWrapper()
 face_areas_z = fields.FaceAreaszWrapper()
 
-print("======== Testing the wrappers of m_edge_lengths =========")
+print("======== Testing the wrappers of edge_lengths =========")
 
 ly_slice_x = edge_lengths_y[nx // 2, :, :]
 lz_slice_x = edge_lengths_z[nx // 2, :, :]
@@ -159,7 +159,7 @@
 print("Perimeter of the middle z-slice:", perimeter_slice_z)
 assert np.isclose(perimeter_slice_z, perimeter_slice_z_true, rtol=1e-05, atol=1e-08)
 
-print("======== Testing the wrappers of m_face_areas =========")
+print("======== Testing the wrappers of face_areas =========")
 
 Sx_slice = np.sum(face_areas_x[nx // 2, :, :])
 dx = (xmax - xmin) / nx

Examples/Tests/magnetostatic_eb/inputs_test_3d_magnetostatic_eb_picmi.py

@@ -225,7 +225,7 @@ def Er_an(r):
 
 er_err = np.abs(Er_mean[r_idx] - Er_an(r_sub)).max() / np.abs(Er_an(r_sub)).max()
 
-plt.ylabel("$E_r$ (V/m)")
+plt.ylabel(r"$E_r$ (V/m)")
 plt.xlabel("r (m)")
 plt.title("Max % Error: {} %".format(er_err * 100.0))
 plt.tight_layout()
@@ -298,7 +298,7 @@ def Bt_an(r):
 
 bt_err = np.abs(Bt_mean[r_idx] - Bt_an(r_sub)).max() / np.abs(Bt_an(r_sub)).max()
 
-plt.ylabel("$B_{\Theta}$ (T)")
+plt.ylabel(r"$B_{\Theta}$ (T)")
 plt.xlabel("r (m)")
 plt.title("Max % Error: {} %".format(bt_err * 100.0))
 plt.tight_layout()