diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7e4edc2dc36..fa14902283c 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -18,6 +18,7 @@ jobs: LAPACKPP_HOME: '/usr/local' OMP_NUM_THREADS: 1 WARPX_CI_CCACHE: 'TRUE' + WARPX_CI_CLEAN_TESTS: 'TRUE' WARPX_CI_NUM_MAKE_JOBS: 2 WARPX_CI_OPENPMD: 'TRUE' WARPX_CI_TMP: '/tmp/ci' @@ -109,6 +110,17 @@ jobs: -DCMAKE_CXX_STANDARD=17 \ -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON fi + if [[ "${WARPX_CI_REGULAR_CARTESIAN_3D:-FALSE}" == "TRUE" ]]; then + cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON + fi rm -rf ${CEI_TMP} df -h displayName: 'Install dependencies' diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml new file mode 100644 index 00000000000..33c6646af73 --- /dev/null +++ b/.github/workflows/clang_sanitizers.yml @@ -0,0 +1,172 @@ +name: 🧴 clang sanitizers + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers + cancel-in-progress: true + +jobs: + build_UB_sanitizer: + name: Clang UB sanitizer + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang15.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-15) + export CC=$(which clang-15) + export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE \ + -DWarpX_PARTICLE_PRECISION=SINGLE + cmake --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with UB sanitizer + run: | + + export OMP_NUM_THREADS=2 + + #MPI implementations often leak memory + export "ASAN_OPTIONS=detect_leaks=0" + + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d + + build_thread_sanitizer: + name: Clang thread sanitizer + runs-on: ubuntu-22.04 + container: ubuntu:23.10 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang17.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-17) + export CC=$(which clang-17) + export CXXFLAGS="-fsanitize=thread" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=OFF \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build -j 4 + + cmake -S . -B build_EB \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="2" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=ON \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build_EB -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with thread sanitizer + run: | + export PMIX_MCA_gds=hash + export TSAN_OPTIONS='ignore_noninstrumented_modules=1' + export ARCHER_OPTIONS="verbose=1" + + # We need these two lines because these tests run inside a docker container + export OMPI_ALLOW_RUN_AS_ROOT=1 + export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + + export OMP_NUM_THREADS=2 + + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d warpx.serialize_initial_conditions = 0 + + git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data + cd Examples/Tests/embedded_circle + + ulimit -c unlimited + + mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_2d warpx.serialize_initial_conditions = 0 + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 8f5e59ccaa4..95d9c148b97 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -43,6 +43,11 @@ jobs: export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} + which nvcc || echo "nvcc not in PATH!" + cmake-easyinstall --prefix=/usr/local \ git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ -DopenPMD_USE_PYTHON=OFF \ @@ -51,6 +56,16 @@ jobs: -DBUILD_CLI_TOOLS=OFF \ -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON + cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | export CCACHE_COMPRESS=1 @@ -71,6 +86,7 @@ jobs: -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 @@ -115,7 +131,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - + cd ../amrex && git checkout --detach 24.08 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/.github/workflows/dependencies/clang17.sh b/.github/workflows/dependencies/clang17.sh new file mode 100755 index 00000000000..d208a9f3f3b --- /dev/null +++ b/.github/workflows/dependencies/clang17.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Copyright 2024 The WarpX Community +# +# License: BSD-3-Clause-LBNL +# Authors: Luca Fedeli + +set -eu -o pipefail + +# This dependency file is currently used within a docker container, +# which does not come with sudo. +apt-get -qqq update +apt-get -y install sudo + +# `man apt.conf`: +# Number of retries to perform. If this is non-zero APT will retry +# failed files the given number of times. +echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries + +# This dependency file is currently used within a docker container, +# which does not come (among others) with wget, xz-utils, curl, git, +# ccache, and pkg-config pre-installed. +sudo apt-get -qqq update +sudo apt-get install -y \ + cmake \ + clang-17 \ + clang-tidy-17 \ + libblas-dev \ + libc++-17-dev \ + libboost-math-dev \ + libfftw3-dev \ + libfftw3-mpi-dev \ + libhdf5-openmpi-dev \ + liblapack-dev \ + libopenmpi-dev \ + libomp-17-dev \ + ninja-build \ + wget \ + xz-utils \ + curl \ + git \ + ccache \ + pkg-config + +# Use clang 17 +export CXX=$(which clang++-17) +export CC=$(which clang-17) + +# cmake-easyinstall +# +sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall +sudo chmod a+x /usr/local/bin/cmake-easyinstall +export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" + +# BLAS++ & LAPACK++ +cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/blaspp.git \ + -Duse_openmp=OFF \ + -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_VERBOSE_MAKEFILE=ON + +cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/lapackpp.git \ + -Duse_cmake_find_lapack=ON \ + -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index ae897786acd..2a1b4d090bc 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -65,6 +65,13 @@ source /etc/profile.d/rocm.sh hipcc --version which clang which clang++ +export CXX=$(which clang++) +export CC=$(which clang) + +# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3 +# https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786 +# https://github.com/open-mpi/ompi/issues/9317 +export LDFLAGS="-lopen-pal" # cmake-easyinstall # @@ -72,3 +79,16 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + +# heFFTe +# +cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0f6710b1405..8b89e3dc4d0 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -55,7 +55,8 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_FFT=ON + -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -115,7 +116,8 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_FFT=ON + -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9f4953b7a98..05e53883534 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -224,61 +224,6 @@ jobs: export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py - build_UB_sanitizer: - name: Clang UB sanitizer - runs-on: ubuntu-22.04 - if: github.event.pull_request.draft == false - env: - CC: clang - CXX: clang++ - # On CI for this test, Ninja is slower than the default: - #CMAKE_GENERATOR: Ninja - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - .github/workflows/dependencies/clang15.sh - - name: CCache Cache - uses: actions/cache@v4 - with: - path: ~/.cache/ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: build WarpX - run: | - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=10 - export CCACHE_MAXSIZE=100M - ccache -z - - export CXX=$(which clang++-15) - export CC=$(which clang-15) - export CXXFLAGS="-fsanitize=undefined -fno-sanitize-recover=all" - - cmake -S . -B build \ - -GNinja \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_FFT=ON \ - -DWarpX_QED=ON \ - -DWarpX_QED_TABLE_GEN=ON \ - -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE \ - -DWarpX_PARTICLE_PRECISION=SINGLE - cmake --build build -j 4 - - ccache -s - du -hs ~/.cache/ccache - - - name: run with UB sanitizer - run: | - export OMP_NUM_THREADS=2 - mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz - mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d - mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d - mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d - save_pr_number: if: github.event_name == 'pull_request' runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c81a057131..4219b33f78b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.07) +project(WarpX VERSION 24.08) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) @@ -711,7 +711,9 @@ endforeach() # pip helpers for the pywarpx package ######################################### # if(WarpX_PYTHON) - set(PYINSTALLOPTIONS "" CACHE STRING + set(PY_PIP_OPTIONS "-v" CACHE STRING + "Additional parameters to pass to `pip`") + set(PY_PIP_INSTALL_OPTIONS "" CACHE STRING "Additional parameters to pass to `pip install`") # ensure all targets are built before we package them in a wheel @@ -735,7 +737,7 @@ if(WarpX_PYTHON) ${CMAKE_COMMAND} -E rm -f -r warpx-whl COMMAND ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$ - ${Python_EXECUTABLE} -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} wheel --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -749,7 +751,7 @@ if(WarpX_PYTHON) set(pyWarpX_REQUIREMENT_FILE "requirements.txt") endif() add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements - ${Python_EXECUTABLE} -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install ${PY_PIP_INSTALL_OPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" WORKING_DIRECTORY ${WarpX_BINARY_DIR} ) @@ -766,7 +768,7 @@ if(WarpX_PYTHON) # because otherwise pip would also force reinstall all dependencies. add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -779,7 +781,7 @@ if(WarpX_PYTHON) # this is for package managers only add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 7997b1e338c..a1ef15b09bf 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.07' +version = u'24.08' # The full version, including alpha/beta/rc tags. -release = u'24.07' +release = u'24.08' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 81d16809d50..2452d074ba1 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -65,6 +65,21 @@ Since this will automatically change the JSON file stored on the repo, make a se git add .json git commit -m "reset benchmark for because ..." --author="Tools " +Automated reset of a list of test benchmarks +-------------------------------------------- + +If you set the environment variable ``export CHECKSUM_RESET=ON`` before running tests that are compared against existing benchmarks, the test analysis will reset the benchmarks to the new values, skipping the comparison. + +With `CTest `__ (coming soon), select the test(s) to reset by `name `__ or `label `__. + +.. code-block:: bash + + # regex filter: matched names + CHECKSUM_RESET=ON ctest --test-dir build -R "Langmuir_multi|LaserAcceleration" + + # ... check and commit changes ... + + Reset a benchmark from the Azure pipeline output on Github ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index fde927c10e1..79f49b5aad1 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -83,7 +83,6 @@ CMake Option Default & Values Descr ``CMAKE_BUILD_TYPE`` RelWithDebInfo/**Release**/Debug `Type of build, symbols & optimizations `__ ``CMAKE_INSTALL_PREFIX`` system-dependent path `Install path prefix `__ ``CMAKE_VERBOSE_MAKEFILE`` ON/**OFF** `Print all compiler commands to the terminal during build `__ -``PYINSTALLOPTIONS`` Additional options for ``pip install``, e.g., ``-v --user`` ``WarpX_APP`` **ON**/OFF Build the WarpX executable application ``WarpX_ASCENT`` ON/**OFF** Ascent in situ visualization ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend @@ -104,6 +103,9 @@ CMake Option Default & Values Descr ``WarpX_QED_TOOLS`` ON/**OFF** Build external tool to generate QED lookup tables (requires PICSAR and Boost) ``WarpX_QED_TABLES_GEN_OMP`` **AUTO**/ON/OFF Enables OpenMP support for QED lookup tables generation ``WarpX_SENSEI`` ON/**OFF** SENSEI in situ visualization +``Python_EXECUTABLE`` (newest found) Path to Python executable +``PY_PIP_OPTIONS`` ``-v`` Additional options for ``pip``, e.g., ``-vvv`` +``PY_PIP_INSTALL_OPTIONS`` Additional options for ``pip install``, e.g., ``--user`` ============================= ============================================ ========================================================= WarpX can be configured in further detail with options from AMReX, which are documented in the AMReX manual: @@ -239,7 +241,7 @@ Developers could now change the WarpX source code and then call the build line a .. tip:: If you do *not* develop with :ref:`a user-level package manager `, e.g., because you rely on a HPC system's environment modules, then consider to set up a virtual environment via `Python venv `__. - Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPYINSTALLOPTIONS="--user"``. + Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPY_PIP_INSTALL_OPTIONS="--user"``. .. _building-pip-python: diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test index adfde819531..ffc1865ee0f 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test @@ -47,7 +47,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_test b/Examples/Tests/laser_injection_from_file/inputs.1d_test index 6124b4552a2..6c392883418 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test b/Examples/Tests/laser_injection_from_file/inputs.2d_test index a3003a4978d..e5814471753 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.2d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.3d_test b/Examples/Tests/laser_injection_from_file/inputs.3d_test index e3096bf92d9..ad8159cb650 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.3d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.3d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.RZ_test b/Examples/Tests/laser_injection_from_file/inputs.RZ_test index fe5bb675212..2a539883fec 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.RZ_test +++ b/Examples/Tests/laser_injection_from_file/inputs.RZ_test @@ -41,7 +41,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test index ca50430c21a..f92440188b7 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test +++ b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test @@ -40,7 +40,7 @@ lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_RZ_laser.profile = from_file lasy_RZ_laser.time_chunk_size = 50 -lasy_RZ_laser.lasy_file_name = "laguerrelaserRZ_00000.h5" +lasy_RZ_laser.lasy_file_name = "diags/laguerrelaserRZ_00000.h5" lasy_RZ_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index 543eb62484a..ea0f323f722 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -68,15 +68,15 @@ m_p = 1.00782503223*scc.m_u # Proton mass m_b = 11.00930536*scc.m_u # Boron 11 mass m_reduced = m_p*m_b/(m_p+m_b) -m_a = 4.00260325413*scc.m_u # Alpha mass -m_be = 7.94748*scc.m_p # Beryllium 8 mass +m_a = 4.00260325413*scc.m_u # Alpha (He4) mass +m_be = (8.0053095729+0.00325283863)*scc.m_u # Be8* mass (3.03 MeV ex. state) Z_boron = 5. Z_proton = 1. E_Gamow = (Z_boron*Z_proton*np.pi*scc.fine_structure)**2*2.*m_reduced*scc.c**2 E_Gamow_MeV = E_Gamow/MeV_to_Joule E_Gamow_keV = E_Gamow/keV_to_Joule -E_fusion = 8.59009*MeV_to_Joule # Energy released during p + B -> alpha + Be -E_decay = 0.0918984*MeV_to_Joule # Energy released during Be -> 2*alpha +E_fusion = 5.55610759*MeV_to_Joule # Energy released during p + B -> alpha + Be* +E_decay = 3.12600414*MeV_to_Joule # Energy released during Be* -> 2*alpha E_fusion_total = E_fusion + E_decay # Energy released during p + B -> 3*alpha ## Checks whether this is the 2D or the 3D test @@ -493,10 +493,13 @@ def check_initial_energy1(data, E_com): energy_alpha3_simulation = energy_alpha_slice[4::6] assert(np.all(is_close(energy_alpha1_simulation, energy_alpha1_theory, rtol=5.e-8))) - assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, rtol=1.e-2)) + ## Check that the max / min value are comparable to the analytical value + ## The minimum value is checked to be within 20 keV of the analytical value + ## The maximum value is checked to be within 1% of the analytical value + assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=1.e-2 )) + assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, atol=3.218e-15 )) + assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=1.e-2 )) + assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, atol=3.218e-15 )) def check_initial_energy2(data): ## In WarpX, the initial momentum of the alphas is computed assuming that the fusion process @@ -578,13 +581,16 @@ def check_initial_energy2(data): assert(is_close(np.amax(energy_alpha1_simulation), max_energy_alpha1, rtol=1.e-2)) assert(is_close(np.amin(energy_alpha1_simulation), min_energy_alpha1, rtol=1.e-2)) - ## Tolerance is quite high below because we don't have a lot of alphas to produce good + ## Check that the max / min value are comparable to the analytical value + ## The minimum value is checked to be within 200 keV of the analytical value + ## The maximum value is checked to be within 5% of the analytical value + ## Tolerance is quite high because we don't have a lot of alphas to produce good ## statistics and an event like alpha1 emitted exactly in direction of proton & alpha2 ## emitted exactly in direction opposite to Beryllium is somewhat rare. - assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, rtol=2.5e-1)) + assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=5e-2 )) + assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, atol=3.218e-14 )) + assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=5e-2 )) + assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, atol=3.218e-14 )) def check_xy_isotropy(data): ## Checks that the alpha particles are emitted isotropically in x and y diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 21b89461e6b..6db203d7f85 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -505,6 +505,11 @@ def set_species_attributes(self, species, layout, source_name): species.add_new_group_attr(source_name, 'uy', self.directed_velocity[1]/constants.c) species.add_new_group_attr(source_name, 'uz', self.directed_velocity[2]/constants.c) + if hasattr(self, 'density_min'): + species.add_new_group_attr(source_name, 'density_min', self.density_min) + if hasattr(self, 'density_max'): + species.add_new_group_attr(source_name, 'density_max', self.density_max) + def setup_parse_momentum_functions(self, species, source_name, expressions, suffix, defaults): for sdir, idir in zip(['x', 'y', 'z'], [0, 1, 2]): if expressions[idir] is not None: @@ -556,6 +561,14 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist Parameters ---------- + warpx_density_min: float + Minimum plasma density. No particle is injected where the density is + below this value. + + warpx_density_max: float + Maximum plasma density. The density at each point is the minimum between + the value given in the profile, and density_max. + warpx_momentum_spread_expressions: list of string Analytic expressions describing the gamma*velocity spread for each axis [m/s]. Expressions should be in terms of the position, written as 'x', 'y', and 'z'. @@ -564,6 +577,8 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist """ def init(self, kw): + self.density_min = kw.pop('warpx_density_min', None) + self.density_max = kw.pop('warpx_density_max', None) self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): diff --git a/Python/setup.py b/Python/setup.py index 31f35eeceac..ed0d4ce29e0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.07', + version = '24.08', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index cdee4b078b0..cf70a750b44 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -2,22 +2,6 @@ "lev=0": { "rho": 0.0 }, - "alpha2": { - "particle_momentum_x": 4.057984510682467e-15, - "particle_momentum_y": 4.104188139725855e-15, - "particle_momentum_z": 4.17858000090827e-15, - "particle_position_x": 408793.7905852193, - "particle_position_y": 861780.8020495367, - "particle_weight": 5.078061191951185e+18 - }, - "alpha3": { - "particle_momentum_x": 5.017656304003558e-16, - "particle_momentum_y": 4.935595075276182e-16, - "particle_momentum_z": 4.867133212376827e-16, - "particle_position_x": 52678.192400911765, - "particle_position_y": 105483.59950020742, - "particle_weight": 1.5413633830148085e+27 - }, "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -26,6 +10,22 @@ "particle_position_y": 81921136.14476715, "particle_weight": 128.00000000000261 }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399486212205656e+30 + }, + "alpha5": { + "particle_momentum_x": 2.3827075626988002e-14, + "particle_momentum_y": 2.388035811027714e-14, + "particle_momentum_z": 2.4040151761604078e-14, + "particle_position_x": 2457556.8571638423, + "particle_position_y": 4914659.635379322, + "particle_weight": 3.839999999999998e-19 + }, "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -34,53 +34,37 @@ "particle_position_y": 819270.9858143466, "particle_weight": 1.2799999998307316e+28 }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 - }, "alpha1": { - "particle_momentum_x": 4.691909092431811e-15, - "particle_momentum_y": 4.6836958163275755e-15, - "particle_momentum_z": 4.657203546376977e-15, + "particle_momentum_x": 4.758865177822356e-15, + "particle_momentum_y": 4.7625215232714745e-15, + "particle_momentum_z": 4.734412976565431e-15, "particle_position_x": 463465.27131109464, "particle_position_y": 978207.6061359186, "particle_weight": 4.977661460251435e-28 }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512788807322e+28 - }, - "proton1": { + "proton5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 }, - "alpha5": { - "particle_momentum_x": 2.3343485859070736e-14, - "particle_momentum_y": 2.3451128753701046e-14, - "particle_momentum_z": 2.3579462789062662e-14, - "particle_position_x": 2457556.8571638423, - "particle_position_y": 4914659.635379322, - "particle_weight": 3.839999999999998e-19 + "proton3": { + "particle_momentum_x": 1.6847290893972251e-15, + "particle_momentum_y": 1.6827074502304075e-15, + "particle_momentum_z": 1.6802489646490975e-15, + "particle_position_x": 2457270.6999197667, + "particle_position_y": 4914315.665267942, + "particle_weight": 1.279486212205663e+30 }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399486212205656e+30 + "alpha2": { + "particle_momentum_x": 4.140109871048478e-15, + "particle_momentum_y": 4.159837925039718e-15, + "particle_momentum_z": 4.251680389225861e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 5.078061191951185e+18 }, "proton4": { "particle_momentum_x": 0.0, @@ -90,28 +74,44 @@ "particle_position_y": 819198.7077771134, "particle_weight": 1.2800000000000004e+37 }, - "proton3": { - "particle_momentum_x": 1.6847290893972251e-15, - "particle_momentum_y": 1.6827074502304075e-15, - "particle_momentum_z": 1.6802489646490975e-15, - "particle_position_x": 2457270.6999197667, - "particle_position_y": 4914315.665267942, - "particle_weight": 1.279486212205663e+30 + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, + "particle_weight": 128.00000000000261 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 }, "alpha4": { - "particle_momentum_x": 2.338084461204216e-14, - "particle_momentum_y": 2.3436156778849828e-14, - "particle_momentum_z": 2.35535708386288e-14, + "particle_momentum_x": 2.3854178157605226e-14, + "particle_momentum_y": 2.3882457581100904e-14, + "particle_momentum_z": 2.403451456378969e-14, "particle_position_x": 2457367.4582781536, "particle_position_y": 4915112.044373056, "particle_weight": 384.0000000000002 }, - "proton5": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512788807322e+28 + }, + "alpha3": { + "particle_momentum_x": 5.079136896829917e-16, + "particle_momentum_y": 5.075922501147803e-16, + "particle_momentum_z": 4.962151258214859e-16, + "particle_position_x": 52678.192400911765, + "particle_position_y": 105483.59950020742, + "particle_weight": 1.5413633830148085e+27 } } diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index ec7e047c537..5aa161dab99 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,17 +1,17 @@ { - "lev=0": { - "rho": 0.0 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "boron1": { + "lev=0": { + "rho": 0.0 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 + }, + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.524872467113344e-13, @@ -19,35 +19,36 @@ "particle_position_y": 40961136.14476712, "particle_position_z": 81920546.19181262, "particle_weight": 1024.000000000021 - }, - "alpha3": { - "particle_momentum_x": 4.764404554793872e-16, - "particle_momentum_y": 4.655900875811434e-16, - "particle_momentum_z": 4.578927372510084e-16, - "particle_position_x": 50987.442011759704, - "particle_position_y": 48999.674675246955, - "particle_position_z": 101142.57224226737, - "particle_weight": 1.0633705344227063e+28 - }, - "alpha1": { - "particle_momentum_x": 4.665933695243743e-15, - "particle_momentum_y": 4.603805875733438e-15, - "particle_momentum_z": 4.706765986105302e-15, + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.1196455431551905e+31 + }, + "alpha2": { + "particle_momentum_x": 4.172264972648105e-15, + "particle_momentum_y": 4.1838781269703316e-15, + "particle_momentum_z": 4.256769380749126e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 3.3372246639840338e+19 + }, + + "alpha1": { + "particle_momentum_x": 4.743550273664842e-15, + "particle_momentum_y": 4.703189539479841e-15, + "particle_momentum_z": 4.75829084832661e-15, "particle_position_x": 461871.79172011977, "particle_position_y": 461162.2166206925, "particle_position_z": 969262.7809050508, "particle_weight": 3.2387855108185994e-27 - }, - "alpha5": { - "particle_momentum_x": 2.3388206254864998e-14, - "particle_momentum_y": 2.334372885765467e-14, - "particle_momentum_z": 2.363588638941874e-14, - "particle_position_x": 2457556.857163843, - "particle_position_y": 2457059.6353793247, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron5": { + }, + "boron5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, @@ -55,17 +56,17 @@ "particle_position_y": 409518.5558814353, "particle_position_z": 819306.5006950963, "particle_weight": 1023.9999999999999 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3745333755307162e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.022781024024315e+29 - }, - "proton4": { + }, + "proton3": { + "particle_momentum_x": 1.6847282386883186e-15, + "particle_momentum_y": 1.6828065767793222e-15, + "particle_momentum_z": 1.6803456707569493e-15, + "particle_position_x": 2457343.371083716, + "particle_position_y": 2457033.3891170574, + "particle_position_z": 4914529.855222688, + "particle_weight": 1.023645543155193e+31 + }, + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 1.7586062624930794e-15, @@ -73,44 +74,44 @@ "particle_position_y": 409598.7077771135, "particle_position_z": 818958.0399127571, "particle_weight": 1.0240000000000003e+38 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.1196455431551905e+31 - }, - "alpha4": { - "particle_momentum_x": 2.33898275612641e-14, - "particle_momentum_y": 2.3423797451957437e-14, - "particle_momentum_z": 2.3516107929259732e-14, + }, + "alpha3": { + "particle_momentum_x": 4.822525301678111e-16, + "particle_momentum_y": 4.78793407068675e-16, + "particle_momentum_z": 4.710411155693663e-16, + "particle_position_x": 50987.442011759704, + "particle_position_y": 48999.674675246955, + "particle_position_z": 101142.57224226737, + "particle_weight": 1.0633705344227063e+28 + }, + "alpha5": { + "particle_momentum_x": 2.3856918226484716e-14, + "particle_momentum_y": 2.3827536503955812e-14, + "particle_momentum_z": 2.405465166862308e-14, + "particle_position_x": 2457556.857163843, + "particle_position_y": 2457059.6353793247, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 + }, + "alpha4": { + "particle_momentum_x": 2.386438069023501e-14, + "particle_momentum_y": 2.3877492448396915e-14, + "particle_momentum_z": 2.4013650859080414e-14, "particle_position_x": 2457367.458278153, "particle_position_y": 2457512.0443730573, "particle_position_z": 4914475.7765130745, "particle_weight": 3072.000000000002 - }, - "proton3": { - "particle_momentum_x": 1.6847282386883186e-15, - "particle_momentum_y": 1.6828065767793222e-15, - "particle_momentum_z": 1.6803456707569493e-15, - "particle_position_x": 2457343.371083716, - "particle_position_y": 2457033.3891170574, - "particle_position_z": 4914529.855222688, - "particle_weight": 1.023645543155193e+31 - }, - "proton5": { + }, + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 - }, - "boron2": { + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.022781024024315e+29 + }, + "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, @@ -118,14 +119,14 @@ "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, "particle_weight": 1.0239999998887592e+29 - }, - "alpha2": { - "particle_momentum_x": 4.1179548991012315e-15, - "particle_momentum_y": 4.110026665992801e-15, - "particle_momentum_z": 4.169802553223462e-15, - "particle_position_x": 408575.75269073684, - "particle_position_y": 413407.5155277014, - "particle_position_z": 863983.4313441743, - "particle_weight": 3.3372246639840338e+19 - } + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 + } } diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index c13f404857a..726573ef8b2 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 381957.7394223898, - "T_ion": 319565.6269784763 + "T_electron": 351982.0169218243, + "T_ion": 349599.6939052666 }, "electron": { - "particle_momentum_x": 8.631833485301232e-19, - "particle_momentum_y": 8.476316745254719e-19, - "particle_momentum_z": 8.514139891331418e-19, - "particle_position_x": 21253105.73686561, - "particle_position_y": 21282643.519070115, - "particle_position_z": 21239057.457948968, + "particle_momentum_x": 8.359982321196841e-19, + "particle_momentum_y": 8.192841151167721e-19, + "particle_momentum_z": 8.182985690701241e-19, + "particle_position_x": 21255110.08090505, + "particle_position_y": 21303488.6242626, + "particle_position_z": 21238676.122703437, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 1.9215585867122464e-18, - "particle_momentum_y": 1.7471481568315848e-18, - "particle_momentum_z": 1.7510887207292533e-18, - "particle_position_x": 21228900.948879313, - "particle_position_y": 21304564.011731848, - "particle_position_z": 21250585.221808463, + "particle_momentum_x": 2.0034830240966893e-18, + "particle_momentum_y": 1.8323959076577197e-18, + "particle_momentum_z": 1.827953230828629e-18, + "particle_position_x": 21246214.748882487, + "particle_position_y": 21280709.710960124, + "particle_position_z": 21206153.002106402, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index f90c34bc86d..4fcf00ced62 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -7,20 +7,20 @@ "Ey": 0.0, "Ez": 0.0 }, - "ion": { - "particle_momentum_x": 2.4932317055825563e-19, - "particle_momentum_y": 2.274916403334278e-19, - "particle_momentum_z": 2.2528161767665816e-19, - "particle_position_x": 2648815.601036139, - "particle_position_y": 2662836.7581390506, + "electron": { + "particle_momentum_x": 1.0618161248729303e-19, + "particle_momentum_y": 1.0331186403682394e-19, + "particle_momentum_z": 1.0375409035564829e-19, + "particle_position_x": 2652982.4592067804, + "particle_position_y": 2666143.4238272114, "particle_weight": 1.7256099431746894e+26 }, - "electron": { - "particle_momentum_x": 1.0489203687862582e-19, - "particle_momentum_y": 1.0209657029567292e-19, - "particle_momentum_z": 1.0248962872393911e-19, - "particle_position_x": 2657004.8285825616, - "particle_position_y": 2670174.272797987, + "ion": { + "particle_momentum_x": 2.4479519290953386e-19, + "particle_momentum_y": 2.2313460312794214e-19, + "particle_momentum_z": 2.207395147435577e-19, + "particle_position_x": 2666525.886108531, + "particle_position_y": 2666683.4040517565, "particle_weight": 1.7256099431746894e+26 } } diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index cc13ceefa28..11adae5b5e1 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -42,6 +42,9 @@ def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. + If the environment variable CHECKSUM_RESET is set while this function is run, + the evaluation will be replaced with a call to reset_benchmark (see below). + Parameters ---------- test_name: string @@ -65,9 +68,17 @@ def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, output_file, output_format, - do_fields=do_fields, do_particles=do_particles) - test_checksum.evaluate(rtol=rtol, atol=atol) + # Reset benchmark? + reset = ( os.getenv('CHECKSUM_RESET', 'False').lower() in + ['true', '1', 't', 'y', 'yes', 'on'] ) + + if reset: + print(f"Environment variable CHECKSUM_RESET is set, resetting benchmark for {test_name}") + reset_benchmark(test_name, output_file, output_format, do_fields, do_particles) + else: + test_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) + test_checksum.evaluate(rtol=rtol, atol=atol) def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 68d59d850a0..7d111741389 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 +branch = 24.08 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6efee41d902..6da53c6b760 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,13 +59,13 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 +branch = 24.08 [source] dir = /home/regtester/AMReX_RegTesting/warpx branch = development cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_PYTHON_IPO=OFF -DpyAMReX_IPO=OFF -# -DPYINSTALLOPTIONS="--disable-pip-version-check" +# -DPY_PIP_INSTALL_OPTIONS="--disable-pip-version-check" # individual problems follow diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index 1d3cb04a8a7..9741d9b667b 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -169,7 +169,7 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) ); // Make local copy of xi, to use on device. - const Real xi_c2 = WarpX::quantum_xi_c2; + const Real xi_c2 = m_quantum_xi_c2; // Apply QED correction to electric field, using temporary arrays. amrex::ParallelFor( diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index d6d5baa5f80..7058372bb9d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -107,6 +107,71 @@ namespace WARPX_ALWAYS_ASSERT_WITH_MESSAGE(vc.allGT(gc), ss_msg.str()); } } + + /** + * \brief Check the requested resources and write performance hints + * + * @param[in] total_nboxes total number of boxes in the simulation + * @param[in] nprocs number of MPI processes + */ + void PerformanceHints (const amrex::Long total_nboxes, const amrex::Long nprocs) + { + // Check: are there more MPI ranks than Boxes? + if (nprocs > total_nboxes) { + std::stringstream warnMsg; + warnMsg << "Too many resources / too little work!\n" + << " It looks like you requested more compute resources than " + << "there are total number of boxes of cells available (" + << total_nboxes << "). " + << "You started with (" << nprocs + << ") MPI ranks, so (" << nprocs - total_nboxes + << ") rank(s) will have no work.\n" + #ifdef AMREX_USE_GPU + << " On GPUs, consider using 1-8 boxes per GPU that together fill " + << "each GPU's memory sufficiently. If you do not rely on dynamic " + << "load-balancing, then one large box per GPU is ideal.\n" + #endif + << "Consider decreasing the amr.blocking_factor and " + << "amr.max_grid_size parameters and/or using fewer MPI ranks.\n" + << " More information:\n" + << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; + + ablastr::warn_manager::WMRecordWarning( + "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + } + + #ifdef AMREX_USE_GPU + // Check: Are there more than 12 boxes per GPU? + if (total_nboxes > nprocs * 12) { + std::stringstream warnMsg; + warnMsg << "Too many boxes per GPU!\n" + << " It looks like you split your simulation domain " + << "in too many boxes (" << total_nboxes << "), which " + << "results in an average number of (" + << amrex::Long(total_nboxes/nprocs) << ") per GPU. " + << "This causes severe overhead in the communication of " + << "border/guard regions.\n" + << " On GPUs, consider using 1-8 boxes per GPU that together fill " + << "each GPU's memory sufficiently. If you do not rely on dynamic " + << "load-balancing, then one large box per GPU is ideal.\n" + << "Consider increasing the amr.blocking_factor and " + << "amr.max_grid_size parameters and/or using more MPI ranks.\n" + << " More information:\n" + << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; + + ablastr::warn_manager::WMRecordWarning( + "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + } + #endif + + // TODO: warn if some ranks have disproportionally more work than all others + // tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down + // all other ranks; we need to measure this with our load-balancing + // routines and issue a warning only of some ranks stall all other ranks + // TODO: check MPI-rank to GPU ratio (should be 1:1) + // TODO: check memory per MPI rank, especially if GPUs are underutilized + // TODO: CPU tiling hints with OpenMP + } } void @@ -534,7 +599,14 @@ WarpX::InitData () } } - PerformanceHints(); + // Computes available boxes on all levels. + amrex::Long total_nboxes = 0; + for (int ilev = 0; ilev <= finestLevel(); ++ilev) { + total_nboxes += boxArray(ilev).size(); + } + auto const nprocs = ParallelDescriptor::NProcs(); + + ::PerformanceHints(total_nboxes, nprocs); CheckKnownIssues(); } @@ -1122,73 +1194,6 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( } } -void -WarpX::PerformanceHints () -{ - // Check requested MPI ranks and available boxes - amrex::Long total_nboxes = 0; // on all MPI ranks - for (int ilev = 0; ilev <= finestLevel(); ++ilev) { - total_nboxes += boxArray(ilev).size(); - } - auto const nprocs = ParallelDescriptor::NProcs(); - - // Check: are there more MPI ranks than Boxes? - if (nprocs > total_nboxes) { - std::stringstream warnMsg; - warnMsg << "Too many resources / too little work!\n" - << " It looks like you requested more compute resources than " - << "there are total number of boxes of cells available (" - << total_nboxes << "). " - << "You started with (" << nprocs - << ") MPI ranks, so (" << nprocs - total_nboxes - << ") rank(s) will have no work.\n" -#ifdef AMREX_USE_GPU - << " On GPUs, consider using 1-8 boxes per GPU that together fill " - << "each GPU's memory sufficiently. If you do not rely on dynamic " - << "load-balancing, then one large box per GPU is ideal.\n" -#endif - << "Consider decreasing the amr.blocking_factor and " - << "amr.max_grid_size parameters and/or using fewer MPI ranks.\n" - << " More information:\n" - << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; - - ablastr::warn_manager::WMRecordWarning( - "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - } - -#ifdef AMREX_USE_GPU - // Check: Are there more than 12 boxes per GPU? - if (total_nboxes > nprocs * 12) { - std::stringstream warnMsg; - warnMsg << "Too many boxes per GPU!\n" - << " It looks like you split your simulation domain " - << "in too many boxes (" << total_nboxes << "), which " - << "results in an average number of (" - << amrex::Long(total_nboxes/nprocs) << ") per GPU. " - << "This causes severe overhead in the communication of " - << "border/guard regions.\n" - << " On GPUs, consider using 1-8 boxes per GPU that together fill " - << "each GPU's memory sufficiently. If you do not rely on dynamic " - << "load-balancing, then one large box per GPU is ideal.\n" - << "Consider increasing the amr.blocking_factor and " - << "amr.max_grid_size parameters and/or using more MPI ranks.\n" - << " More information:\n" - << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; - - ablastr::warn_manager::WMRecordWarning( - "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - } -#endif - - // TODO: warn if some ranks have disproportionally more work than all others - // tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down - // all other ranks; we need to measure this with our load-balancing - // routines and issue a warning only of some ranks stall all other ranks - // TODO: check MPI-rank to GPU ratio (should be 1:1) - // TODO: check memory per MPI rank, especially if GPUs are underutilized - // TODO: CPU tiling hints with OpenMP -} - void WarpX::CheckGuardCells() { for (int lev = 0; lev <= finest_level; ++lev) diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 375cc1e6d51..63f35d3d254 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -33,11 +33,11 @@ namespace { * \brief This function initializes the momentum of the alpha particles produced from * proton-boron fusion. The momentum is initialized by assuming that the fusion of a proton * with a boron nucleus into 3 alphas takes place in two steps. In the first step, the proton - * and the boron fuse into a beryllium nucleus and an alpha particle. In the second step, the - * beryllium decays into two alpha particles. The first step produces 8.59009 MeV of kinetic - * energy while the second step produces 91.8984 keV of kinetic energy. This two-step process + * and the boron fuse into an excited beryllium nucleus and an alpha particle. In the second step, the + * excited beryllium decays into two alpha particles. The first step produces ~5.56 MeV of kinetic + * energy while the second step produces ~3.12 MeV of kinetic energy. This two-step process * is considered to be the dominant process of proton+boron fusion into alphas (see - * Becker et al., Zeitschrift für Physik A Atomic Nuclei, 327(3), 341-355 (1987)). + * D.R. Tilley et al. / Nuclear Physics A 745 (2004) 155–362). * For each step, we assume in this function that the particles are emitted isotropically in * the corresponding center of mass frame (center of mass frame of proton + boron for the * creation of first alpha+beryllium and rest frame of beryllium for the creation of second and @@ -73,18 +73,21 @@ namespace { constexpr amrex::ParticleReal mev_to_joule = PhysConst::q_e*1.e6_prt; // Energy produced in the fusion reaction proton + boron11 -> Beryllium8 + alpha - // cf. Janis book of proton-induced cross-sections (2019) - constexpr amrex::ParticleReal E_fusion = 8.59009_prt*mev_to_joule; + // cf. https://doi.org/10.1016/j.radphyschem.2022.110727 + // Dominant reaction channel is p + B11 -> alpha_1 + 8Be* -> alpha_1 + alpha_11 + alpha_12 + 8.68 MeV + // cf. Kelley et al., (2017) http://dx.doi.org/10.1016/j.nuclphysa.2017.07.015 + constexpr amrex::ParticleReal E_fusion = 5.55610759_prt*mev_to_joule; // Energy produced when Beryllium8 decays into two alphas - // cf. JEFF-3.3 radioactive decay data library (2017) - constexpr amrex::ParticleReal E_decay = 0.0918984_prt*mev_to_joule; + // cf. Kelley et al., (2017) http://dx.doi.org/10.1016/j.nuclphysa.2017.07.015 + constexpr amrex::ParticleReal E_decay = 3.12600414_prt*mev_to_joule; // The constexprs ma_sq and mBe_sq underflow in single precision because we use SI units, // which can cause compilation to fail or generate a warning, so we're explicitly setting // them as double. Note that nuclear fusion module does not currently work with single // precision anyways. constexpr double m_alpha = PhysConst::m_u * 4.00260325413_prt; - constexpr double m_beryllium = PhysConst::m_p * 7.94748_prt; + // mass of the Be8 excited state (3.03 MeV above ground state) + constexpr double m_beryllium = PhysConst::m_u*(8.0053095729_prt+0.00325283863_prt); constexpr double mBe_sq = m_beryllium*m_beryllium; constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 25068b2e65c..7b2ebce92b9 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -401,6 +401,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC for (int lev = 0; lev < pc.numLevels(); ++lev) { + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + const auto& plevel = pc.GetParticles(lev); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -408,10 +413,10 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) { continue; } - auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( - lev, pti.index(), pti.LocalTileIndex()); + auto& ptile_buffer = + species_buffer.ParticlesAt(lev, pti.index(), pti.LocalTileIndex()); + const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); if (np == 0) { continue; } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index d4c325fbfb8..b44cb33d66b 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -433,7 +433,7 @@ protected: amrex::ParticleReal charge; amrex::ParticleReal mass; - PhysicalSpecies physical_species; + PhysicalSpecies physical_species = PhysicalSpecies::unspecified; // Controls boundaries for particles exiting the domain ParticleBoundaries m_boundary_conditions; diff --git a/Source/WarpX.H b/Source/WarpX.H index 3d3878baca4..4aad7ba5349 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -233,7 +233,7 @@ public: //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation - bool current_correction; + bool current_correction = true; //! If true, the PSATD update equation for E contains both J and rho //! (default is false for standard PSATD and true for Galilean PSATD) @@ -671,7 +671,7 @@ public: */ void Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real dt); - static amrex::Real quantum_xi_c2; + amrex::Real m_quantum_xi_c2; /** Check and potentially compute load balancing */ @@ -1405,9 +1405,6 @@ private: */ void CheckKnownIssues(); - /** Check the requested resources and write performance hints */ - void PerformanceHints (); - void BuildBufferMasks (); [[nodiscard]] const amrex::iMultiFab* getCurrentBufferMasks (int lev) const { diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 3ae9fd18106..fe79ce9c771 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -100,7 +100,6 @@ bool WarpX::fft_do_time_averaging = false; amrex::IntVect WarpX::m_fill_guards_fields = amrex::IntVect(0); amrex::IntVect WarpX::m_fill_guards_current = amrex::IntVect(0); -Real WarpX::quantum_xi_c2 = PhysConst::xi_c2; Real WarpX::gamma_boost = 1._rt; Real WarpX::beta_boost = 0._rt; Vector WarpX::boost_direction = {0,0,0}; @@ -905,12 +904,15 @@ WarpX::ReadParameters () utils::parser::queryWithParser( pp_warpx, "n_current_deposition_buffer", n_current_deposition_buffer); + //Default value for the quantum parameter used in Maxwell’s QED equations + m_quantum_xi_c2 = PhysConst::xi_c2; + amrex::Real quantum_xi_tmp; const auto quantum_xi_is_specified = utils::parser::queryWithParser(pp_warpx, "quantum_xi", quantum_xi_tmp); if (quantum_xi_is_specified) { double const quantum_xi = quantum_xi_tmp; - quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); + m_quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); } const auto at_least_one_boundary_is_pml = @@ -1546,7 +1548,6 @@ WarpX::ReadParameters () // Current correction activated by default, unless a charge-conserving // current deposition (Esirkepov, Vay) or the div(E) cleaning scheme // are used - current_correction = true; if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor || WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay || diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index 9ceb91457c9..07a2d9d2732 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -42,8 +42,11 @@ namespace ablastr::math::anyfft } else if (dim == 2) { result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorR2C); + } else if (dim == 1) { + result = cufftPlan1d( + &(fft_plan.m_plan), real_size[0], VendorR2C, 1); } else { - ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); + ABLASTR_ABORT_WITH_MESSAGE("only dim=1 and dim=2 and dim=3 have been implemented"); } } else { if (dim == 3) { @@ -52,6 +55,10 @@ namespace ablastr::math::anyfft } else if (dim == 2) { result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorC2R); + } else if (dim == 1) { + int batch = 2; + result = cufftPlan1d( + &(fft_plan.m_plan), real_size[0], VendorC2R, 1); } else { ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); } diff --git a/Source/ablastr/math/fft/WrapFFTW.cpp b/Source/ablastr/math/fft/WrapFFTW.cpp index 6711bbface9..16f0355cc5d 100644 --- a/Source/ablastr/math/fft/WrapFFTW.cpp +++ b/Source/ablastr/math/fft/WrapFFTW.cpp @@ -25,11 +25,15 @@ namespace ablastr::math::anyfft const auto VendorCreatePlanC2R3D = fftwf_plan_dft_c2r_3d; const auto VendorCreatePlanR2C2D = fftwf_plan_dft_r2c_2d; const auto VendorCreatePlanC2R2D = fftwf_plan_dft_c2r_2d; + const auto VendorCreatePlanR2C1D = fftwf_plan_dft_r2c_1d; + const auto VendorCreatePlanC2R1D = fftwf_plan_dft_c2r_1d; #else const auto VendorCreatePlanR2C3D = fftw_plan_dft_r2c_3d; const auto VendorCreatePlanC2R3D = fftw_plan_dft_c2r_3d; const auto VendorCreatePlanR2C2D = fftw_plan_dft_r2c_2d; const auto VendorCreatePlanC2R2D = fftw_plan_dft_c2r_2d; + const auto VendorCreatePlanR2C1D = fftw_plan_dft_r2c_1d; + const auto VendorCreatePlanC2R1D = fftw_plan_dft_c2r_1d; #endif FFTplan CreatePlan(const amrex::IntVect& real_size, amrex::Real * const real_array, @@ -56,9 +60,12 @@ namespace ablastr::math::anyfft } else if (dim == 2) { fft_plan.m_plan = VendorCreatePlanR2C2D( real_size[1], real_size[0], real_array, complex_array, FFTW_ESTIMATE); + } else if (dim == 1) { + fft_plan.m_plan = VendorCreatePlanR2C1D( + real_size[0], real_array, complex_array, FFTW_ESTIMATE); } else { ABLASTR_ABORT_WITH_MESSAGE( - "only dim=2 and dim=3 have been implemented"); + "only dim=1 and dim=2 and dim=3 have been implemented"); } } else if (dir == direction::C2R){ if (dim == 3) { @@ -67,9 +74,12 @@ namespace ablastr::math::anyfft } else if (dim == 2) { fft_plan.m_plan = VendorCreatePlanC2R2D( real_size[1], real_size[0], complex_array, real_array, FFTW_ESTIMATE); + } else if (dim == 1) { + fft_plan.m_plan = VendorCreatePlanC2R1D( + real_size[0], complex_array, real_array, FFTW_ESTIMATE); } else { ABLASTR_ABORT_WITH_MESSAGE( - "only dim=2 and dim=3 have been implemented. Should be easy to add dim=1."); + "only dim=1 and dim=2 and dim=3 have been implemented."); } } diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index f1e5b3f62e2..39d88010517 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.07 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.08 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "20e6f2eadf0c297517588ba38973ec7c7084fa31" +set(WarpX_amrex_branch "24.08" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 133747aaeae..020502d9921 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.07 CONFIG REQUIRED) + find_package(pyAMReX 24.08 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "e007e730d48cb5fdbe1e10462d7d0a14e2bc8f8e" +set(WarpX_pyamrex_branch "24.08" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index f56a957c17a..11360673b63 100755 --- a/run_test.sh +++ b/run_test.sh @@ -16,8 +16,10 @@ # physically correct. # The tests can be influenced by environment variables: +# Use `export WARPX_CI_CLEAN_TESTS=ON` in order to remove all subdirectories +# from each test directory, directly after a test has passed. # Use `export WARPX_CI_DIM=3` or `export WARPX_CI_DIM=2` in order to -# select only the tests that correspond to this dimension +# select only the tests that correspond to this dimension. # Use `export WARPX_TEST_ARCH=CPU` or `export WARPX_TEST_ARCH=GPU` in order # to run the tests on CPU or GPU respectively. @@ -30,6 +32,7 @@ tests_arg=$* tests_run=${tests_arg:+--tests=${tests_arg}} # environment options +WARPX_CI_CLEAN_TESTS=${WARPX_CI_CLEAN_TESTS:-""} WARPX_CI_TMP=${WARPX_CI_TMP:-""} # Remove contents and link to a previous test directory (intentionally two arguments) @@ -65,10 +68,11 @@ python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade cmake python3 -m pip install --upgrade -r warpx/Regression/requirements.txt +python3 -m pip cache purge # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - +cd amrex && git checkout --detach 24.08 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets @@ -92,12 +96,17 @@ cp -r Checksum ../../regression_testing/ # Run tests cd ../../regression_testing/ echo "cd $PWD" +if [ -z "${WARPX_CI_CLEAN_TESTS}" ]; then + test_rm_dir="" +else + test_rm_dir="--clean_testdir" +fi # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then - python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all "${tests_run}" + python3 regtest.py ../rt-WarpX/ci-tests.ini ${test_rm_dir} --skip_comparison --no_update all "${tests_run}" # run all tests (variables tests_arg and tests_run are empty) else - python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all + python3 regtest.py ../rt-WarpX/ci-tests.ini ${test_rm_dir} --skip_comparison --no_update all fi # clean up python virtual environment diff --git a/setup.py b/setup.py index f05d5a71899..30f91d8fb4a 100644 --- a/setup.py +++ b/setup.py @@ -280,7 +280,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.07', + version = '24.08', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.',