diff --git a/.clang-tidy b/.clang-tidy index 21f0343519c..04d1419c5c7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -7,6 +7,11 @@ Checks: ' -bugprone-unchecked-optional-access, cert-*, -cert-err58-cpp, + clang-analyzer-*, + -clang-analyzer-optin.performance.Padding, + -clang-analyzer-optin.mpi.MPI-Checker, + -clang-analyzer-osx.*, + -clang-analyzer-optin.osx.*, clang-diagnostic-*, cppcoreguidelines-*, -cppcoreguidelines-avoid-c-arrays, @@ -30,8 +35,6 @@ Checks: ' -modernize-use-trailing-return-type, mpi-*, performance-*, - -performance-unnecessary-copy-initialization, - -performance-unnecessary-value-param, portability-*, readability-*, -readability-convert-member-functions-to-static, diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 5e3a3077f5d..de1c8237021 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.03 && cd - + cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2225e670bb0..ae897786acd 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -12,12 +12,24 @@ set -eu -o pipefail # failed files the given number of times. echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries -# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu -wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key \ - | sudo apt-key add - -echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \ - | sudo tee /etc/apt/sources.list.d/rocm.list +# Ref.: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/ubuntu.html + +# Make the directory if it doesn't exist yet. +# This location is recommended by the distribution maintainers. +sudo mkdir --parents --mode=0755 /etc/apt/keyrings + +# Download the key, convert the signing-key to a full +# keyring required by apt and store in the keyring directory +wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ + gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null +curl -O https://repo.radeon.com/rocm/rocm.gpg.key +sudo apt-key add rocm.gpg.key + +source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ... + +echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \ + | sudo tee /etc/apt/sources.list.d/rocm.list echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ | sudo tee -a /etc/profile.d/rocm.sh diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index 74f0db4b46a..fae6e22d45a 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -58,7 +58,7 @@ export CEI_TMP="/tmp/cei" CXX=$(which icpc) CC=$(which icc) \ cmake-easyinstall \ --prefix=/usr/local \ - git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ + git+https://github.com/openPMD/openPMD-api.git@0.15.2 \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f34f9f3534d..21d5ae04faf 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -36,9 +36,6 @@ jobs: brew tap openpmd/openpmd brew install openpmd-api - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade virtualenv - python3 -m venv py-venv source py-venv/bin/activate python3 -m pip install --upgrade pip diff --git a/.github/workflows/scripts/checkQEDTableGenerator.sh b/.github/workflows/scripts/checkQEDTableGenerator.sh new file mode 100755 index 00000000000..e14a7a2d6f2 --- /dev/null +++ b/.github/workflows/scripts/checkQEDTableGenerator.sh @@ -0,0 +1,77 @@ + +#!/usr/bin/env bash +# +# Copyright 2022 Luca Fedeli +# +# License: BSD-3-Clause-LBNL + +set -eu -o pipefail + +export OMP_NUM_THREADS=2 + +# +# Generate QED lookup tables using external tool +# +./build/bin/qed_table_generator \ + --table BW --mode DP --dndt_chi_min 0.01 \ + --dndt_chi_max 100 --dndt_how_many 64 \ + --pair_chi_min 0.01 --pair_chi_max 100 \ + --pair_chi_how_many 64 --pair_frac_how_many 64 \ + -o bw_table_tool +./build/bin/qed_table_generator \ + --table QS --mode DP --dndt_chi_min 0.001 \ + --dndt_chi_max 100 --dndt_how_many 64 \ + --em_chi_min 0.001 --em_chi_max 100 \ + --em_chi_how_many 64 --em_frac_how_many 64 \ + --em_frac_min 1e-12 -o qs_table_tool + +# +# Generate QED lookup tables using WarpX +# +./build/bin/warpx.2d \ + ./Examples/Tests/qed/quantum_synchrotron/inputs_2d \ + qed_bw.lookup_table_mode = "generate" \ + qed_bw.tab_dndt_chi_min = 0.01 \ + qed_bw.tab_dndt_chi_max = 100.0 \ + qed_bw.tab_dndt_how_many = 64 \ + qed_bw.tab_pair_chi_min = 0.01 \ + qed_bw.tab_pair_chi_max = 100.0 \ + qed_bw.tab_pair_chi_how_many = 64 \ + qed_bw.tab_pair_frac_how_many = 64 \ + qed_bw.save_table_in = "bw_table" \ + qed_qs.lookup_table_mode = "generate" \ + qed_qs.tab_dndt_chi_min = 0.001 \ + qed_qs.tab_dndt_chi_max = 100.0 \ + qed_qs.tab_dndt_how_many = 64 \ + qed_qs.tab_em_chi_min = 0.001 \ + qed_qs.tab_em_frac_min = 1.0e-12 \ + qed_qs.tab_em_chi_max = 100.0 \ + qed_qs.tab_em_chi_how_many = 64 \ + qed_qs.tab_em_frac_how_many = 64 \ + qed_qs.save_table_in = "qs_table" + +# +# Convert lookup tables (generated with WarpX and with the external tool) in human-readable format +# +./build/bin/qed_table_reader -i qs_table --table QS --mode DP -o qs_table +./build/bin/qed_table_reader -i qs_table_tool --table QS --mode DP -o qs_table_tool +./build/bin/qed_table_reader -i bw_table --table BW --mode DP -o bw_table +./build/bin/qed_table_reader -i bw_table_tool --table BW --mode DP -o bw_table_tool + +# +# Compare the generated lookup tables +# +diff bw_table_dndt bw_table_tool_dndt +diff bw_table_pair bw_table_tool_pair +diff qs_table_phot_em qs_table_tool_phot_em +diff qs_table_dndt qs_table_tool_dndt + +# +# Run a WarpX simulation using the lookup tables generated by the external tool +# +./build/bin/warpx.2d \ + ./Examples/Tests/qed/quantum_synchrotron/inputs_2d \ + qed_bw.lookup_table_mode = "load" \ + qed_bw.load_table_from = "bw_table_tool" \ + qed_qs.lookup_table_mode = "load" \ + qed_qs.load_table_from = "qs_table_tool" diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index cf4b375ce00..2997e9cdd16 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -46,7 +46,7 @@ jobs: du -hs ~/.cache/ccache build_1D_2D: - name: GCC 1D & 2D w/ MPI + name: GCC 1D & 2D w/ MPI, QED tools runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false env: @@ -78,7 +78,9 @@ jobs: -DWarpX_DIMS="1;2" \ -DWarpX_EB=OFF \ -DWarpX_PSATD=ON \ - -DWarpX_QED_TABLE_GEN=ON + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_QED_TOOLS=ON + cmake --build build -j 4 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d @@ -86,6 +88,10 @@ jobs: ccache -s du -hs ~/.cache/ccache + - name: run QED table tools + run: | + .github/workflows/scripts/checkQEDTableGenerator.sh + build_3D_sp: name: GCC 3D & RZ w/ MPI, single precision runs-on: ubuntu-22.04 @@ -218,6 +224,61 @@ jobs: export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + build_UB_sanitizer: + name: Clang UB sanitizer + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang15.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-15) + export CC=$(which clang-15) + export CXXFLAGS="-fsanitize=undefined -fno-sanitize-recover=all" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_PSATD=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE \ + -DWarpX_PARTICLE_PRECISION=SINGLE + cmake --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with UB sanitizer + run: | + export OMP_NUM_THREADS=2 + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d + save_pr_number: if: github.event_name == 'pull_request' runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3d14c123c1..cef24aed2cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] @@ -80,6 +80,7 @@ repos: hooks: - id: isort name: isort (python) + args: ['--profile black'] # Python: Flake8 (checks only, does this support auto-fixes?) #- repo: https://github.com/PyCQA/flake8 diff --git a/.zenodo.json b/.zenodo.json index bbba267def9..a2c9b443f99 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -20,6 +20,11 @@ "name": "Andriyash, Igor", "orcid": "0000-0003-0313-4496" }, + { + "affiliation": "Lawrence Livermore National Laboratory", + "name": "Angus, Justin Ray", + "orcid": "0000-0003-1474-0002" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Belkin, Daniel", @@ -93,6 +98,11 @@ "name": "Gu, Junmin", "orcid": "0000-0002-1521-8534" }, + { + "affiliation": "Lawrence Berkeley National Laboratory", + "name": "Haseeb, Muhammad", + "orcid": "0000-0002-0697-6894" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Jambunathan, Revathi", @@ -155,7 +165,7 @@ }, { "affiliation": "Lawrence Berkeley National Laboratory", - "name": "Sandberg, Ryan T.", + "name": "Sandberg, Ryan Thor", "orcid": "0000-0001-7680-8733" }, { @@ -163,11 +173,21 @@ "name": "Scherpelz, Peter", "orcid": "0000-0001-8185-3387" }, + { + "affiliation": "Laboratory for Laser Energetics, University of Rochester", + "name": "Weichman, Kale", + "orcid": "0000-0002-3487-7922" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Yang, Eloise", "orcid": "0000-0002-9319-4216" }, + { + "affiliation": "LIDYL, CEA-Universit\u00e9 Paris-Saclay, CEA Saclay", + "name": "Zaim, Ne\u00efl", + "orcid": "0000-0003-0313-4496" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Zhang, Weiqun", @@ -187,11 +207,6 @@ "affiliation": "Lawrence Berkeley National Laboratory", "name": "Zoni, Edoardo", "orcid": "0000-0001-5662-4646" - }, - { - "affiliation": "LIDYL, CEA-Universit\u00e9 Paris-Saclay, CEA Saclay", - "name": "Zaim, Ne\u00efl", - "orcid": "0000-0003-0313-4496" } ], "contributors": [ diff --git a/CMakeLists.txt b/CMakeLists.txt index e19a1f3bb4f..8efd9e137bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.03) +project(WarpX VERSION 24.05) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) @@ -69,17 +69,17 @@ include(CMakeDependentOption) option(WarpX_APP "Build the WarpX executable application" ON) option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) option(WarpX_EB "Embedded boundary support" OFF) -cmake_dependent_option(WarpX_GPUCLOCK - "Add GPU kernel timers (cost function)" ON - "WarpX_COMPUTE STREQUAL CUDA OR WarpX_COMPUTE STREQUAL HIP" OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_PSATD "spectral solver support" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) -option(WarpX_QED "QED support (requires PICSAR)" ON) -option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) +option(WarpX_QED "QED support (requires PICSAR)" ON) +option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" + OFF) +option(WarpX_QED_TOOLS "Build external tool to generate QED lookup tables (requires PICSAR and Boost)" + OFF) set(WarpX_DIMS_VALUES 1 2 3 RZ) set(WarpX_DIMS 3 CACHE STRING "Simulation dimensionality <1;2;3;RZ>") @@ -111,6 +111,13 @@ if(NOT WarpX_PARTICLE_PRECISION IN_LIST WarpX_PARTICLE_PRECISION_VALUES) message(FATAL_ERROR "WarpX_PARTICLE_PRECISION (${WarpX_PARTICLE_PRECISION}) must be one of ${WarpX_PARTICLE_PRECISION_VALUES}") endif() +set(WarpX_QED_TABLES_GEN_OMP_VALUES AUTO ON OFF) +set(WarpX_QED_TABLES_GEN_OMP AUTO CACHE STRING "Enables OpenMP support for QED lookup tables generation (AUTO/ON/OFF)") +set_property(CACHE WarpX_QED_TABLES_GEN_OMP PROPERTY STRINGS ${WarpX_QED_TABLES_GEN_OMP_VALUES}) +if(NOT WarpX_QED_TABLES_GEN_OMP IN_LIST WarpX_QED_TABLES_GEN_OMP_VALUES) + message(FATAL_ERROR "WarpX_QED_TABLES_GEN_OMP (${WarpX_QED_TABLES_GEN_OMP}) must be one of ${WarpX_QED_TABLES_GEN_OMP_VALUES}") +endif() + set(WarpX_COMPUTE_VALUES NOACC OMP CUDA SYCL HIP) set(WarpX_COMPUTE OMP CACHE STRING "On-node, accelerated computing backend (NOACC/OMP/CUDA/SYCL/HIP)") set_property(CACHE WarpX_COMPUTE PROPERTY STRINGS ${WarpX_COMPUTE_VALUES}) @@ -402,6 +409,9 @@ if(WarpX_LIB) add_subdirectory(Source/Python) add_subdirectory(Source/Utils) endif() +if(WarpX_QED_TOOLS) + add_subdirectory(Tools/QedTablesUtils) +endif() # Interprocedural optimization (IPO) / Link-Time Optimization (LTO) if(WarpX_IPO) @@ -499,10 +509,6 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_RZ WARPX_ZINDEX=1) endif() - if(WarpX_GPUCLOCK) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_GPUCLOCK) - endif() - if(WarpX_OPENPMD) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_OPENPMD) endif() diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index 79bb9fa8884..4c7a2d8137c 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -53,6 +53,11 @@ Prior WarpX references If your project uses a specific algorithm or component, please consider citing the respective publications in addition. +- Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint __` + - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 1a439c8b76e..471f6f2b6a3 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -31,8 +31,8 @@ import urllib.request import pybtex.plugin -from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme +from pybtex.style.formatting.unsrt import Style as UnsrtStyle sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.03' +version = u'24.05' # The full version, including alpha/beta/rc tags. -release = u'24.03' +release = u'24.05' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Docs/source/developers/run_clang_tidy_locally.rst b/Docs/source/developers/run_clang_tidy_locally.rst new file mode 100644 index 00000000000..3f600019fe7 --- /dev/null +++ b/Docs/source/developers/run_clang_tidy_locally.rst @@ -0,0 +1,55 @@ +.. _developers-run_clang_tidy_locally: + +The clang-tidy linter +===================== + +Clang-tidy CI test +------------------ + +WarpX's CI tests include several checks performed with the +`clang-tidy `__ linter +(currently the version 15 of this tool). The complete list of checks +enforced in CI tests can be found in the ``.clang-tidy`` configuration file. + +.. dropdown:: clang-tidy configuration file + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../.clang-tidy + :language: yaml + +Run clang-tidy linter locally +----------------------------- + +We provide a script to run clang-tidy locally. The script can be run as follows, +provided that all the requirements to compile WarpX are met (see `building from source `). +The script generates a simple wrapper to ensure that `clang-tidy` is only applied to WarpX source files +and compiles WarpX in 1D,2D,3D, and RZ using such wrapper. By default WarpX is compiled in single precision +with PSATD solver, QED module, QED table generator and Embedded boundary in order to find more +potential issues with the `clang-tidy` tool. + +Few optional environment variables can be set to tune the behavior of the script: + +* ``WARPX_TOOLS_LINTER_PARALLEL``: sets the number of cores to be used for the compilation +* ``CLANG``, ``CLANGXX``, and ``CLANGTIDY`` : set the version of the compiler and of the linter + +Note: clang v15 is currently used in CI tests. It is therefore recommended to use this version. +Otherwise, a newer version may find issues not currently covered by CI tests (checks are opt-in) +while older versions may not find all the issues. + +.. code-block:: bash + + export WARPX_TOOLS_LINTER_PARALLEL=12 + export CLANG=clang-15 + export CLANGXX=clang++-15 + export CLANGTIDY=clang-tidy-15 + ./Tools/Linter/runClangTidy.sh + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../Tools/Linter/runClangTidy.sh + :language: bash diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 6dcc1886e12..c6a09d970df 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -90,7 +90,7 @@ The content of this directory will look like the following (possibly including b $ ls ./test_dir/rt-WarpX/WarpX-tests/2021-04-30/pml_x_yee/ analysis_pml_yee.py # Python analysis script inputs_2d # input file - main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.GPUCLOCK.ex # executable + main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.ex # executable pml_x_yee.analysis.out # Python analysis output pml_x_yee.err.out # error output pml_x_yee.make.out # build output diff --git a/Docs/source/developers/workflows.rst b/Docs/source/developers/workflows.rst index 8b357f5d4da..00279018e9d 100644 --- a/Docs/source/developers/workflows.rst +++ b/Docs/source/developers/workflows.rst @@ -11,3 +11,4 @@ Workflows documentation checksum local_compile + run_clang_tidy_locally diff --git a/Docs/source/governance.rst b/Docs/source/governance.rst new file mode 120000 index 00000000000..1b1b99778ab --- /dev/null +++ b/Docs/source/governance.rst @@ -0,0 +1 @@ +../../GOVERNANCE.rst \ No newline at end of file diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 98b0d85391e..b243f62cd97 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -24,6 +24,11 @@ Scientific works in laser-plasma and beam-plasma acceleration. Phys. Rev. Research **5**, 033112, 2023 `DOI:10.1103/PhysRevResearch.5.033112 `__ +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint `__ + #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. @@ -96,6 +101,11 @@ Particle Accelerator & Beam Physics Scientific works in particle and beam modeling. +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint `__ + #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. @@ -128,6 +138,7 @@ Microelectronics **Characterization of Transmission Lines in Microelectronic Circuits Using the ARTEMIS Solver**. IEEE Journal on Multiscale and Multiphysics Computational Techniques, vol. 8, pp. 31-39, 2023. `DOI:10.1109/JMMCT.2022.3228281 `__ + #. Kumar P, Nonaka A, Jambunathan R, Pahwa G and Salahuddin S, Yao Z. **FerroX: A GPU-accelerated, 3D Phase-Field Simulation Framework for Modeling Ferroelectric Devices**. arXiv preprint, 2022. diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 9f2d2ff038e..89bc2e417cb 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -150,4 +150,5 @@ Epilogue :hidden: glossary + governance acknowledgements diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 4474b3509fe..006cd843b31 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -89,7 +89,6 @@ CMake Option Default & Values Descr ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WarpX_DIMS`` **3**/2/1/RZ Simulation dimensionality. Use ``"1;2;RZ;3"`` for all. ``WarpX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) -``WarpX_GPUCLOCK`` **ON**/OFF Add GPU kernel timers (cost function, +4 registers/kernel) ``WarpX_IPO`` ON/**OFF** Compile WarpX with interprocedural optimization (aka LTO) ``WarpX_LIB`` ON/**OFF** Build WarpX as a library, e.g., for PICMI Python ``WarpX_MPI`` **ON**/OFF Multi-node support (message-passing) @@ -101,6 +100,8 @@ CMake Option Default & Values Descr ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) +``WarpX_QED_TOOLS`` ON/**OFF** Build external tool to generate QED lookup tables (requires PICSAR and Boost) +``WarpX_QED_TABLES_GEN_OMP`` **AUTO**/ON/OFF Enables OpenMP support for QED lookup tables generation ``WarpX_SENSEI`` ON/**OFF** SENSEI in situ visualization ============================= ============================================ ========================================================= @@ -124,7 +125,7 @@ CMake Option Default & Values Des ``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` ``WarpX_openpmd_src`` *None* Path to openPMD-api source directory (preferred if set) ``WarpX_openpmd_repo`` ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from -``WarpX_openpmd_branch`` ``0.15.1`` Repository branch for ``WarpX_openpmd_repo`` +``WarpX_openpmd_branch`` ``0.15.2`` Repository branch for ``WarpX_openpmd_repo`` ``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD-api library if set to ``OFF`` ``WarpX_picsar_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 94b2be78bec..ce9f9dca520 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -63,8 +63,8 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda update -n base conda - conda install -n base conda-libmamba-solver + conda update -y -n base conda + conda install -y -n base conda-libmamba-solver conda config --set solver libmamba We recommend to deactivate that conda self-activates its ``base`` environment. @@ -80,7 +80,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON @@ -90,7 +90,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv + conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api openpmd-viewer python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv conda activate warpx-cpu-dev # compile WarpX with -DWarpX_MPI=OFF diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index a7b0f636b56..5efdeae39a8 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -51,6 +51,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/spock hpc/summit hpc/taurus + hpc/greatlakes .. tip:: diff --git a/Docs/source/install/hpc/greatlakes.rst b/Docs/source/install/hpc/greatlakes.rst new file mode 100644 index 00000000000..d238d498e5c --- /dev/null +++ b/Docs/source/install/hpc/greatlakes.rst @@ -0,0 +1,240 @@ +.. _building-greatlakes: + +Great Lakes (UMich) +=================== + +The `Great Lakes cluster `_ is located at University of Michigan. +The cluster has various partitions, including `GPU nodes and CPU nodes `__. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Great Lakes user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ (`documentation `__) +* `Filesystems `__: + + * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (80GB) + * ``/scratch``: per-project `production directory `__; very fast for parallel jobs; purged every 60 days (10TB default) + + +.. _building-greatlakes-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Great Lakes, you can run either on GPU nodes with `fast V100 GPUs (recommended), the even faster A100 GPUs (only a few available) or CPU nodes `__. + +.. tab-set:: + + .. tab-item:: V100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/greatlakes_v100_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example $HOME/greatlakes_v100_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``iloveplasma``, then run ``nano $HOME/greatlakes_v100_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="iloveplasma" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to Great Lakes, activate these environment settings: + + .. code-block:: bash + + source $HOME/greatlakes_v100_warpx.profile + + Finally, since Great Lakes does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/greatlakes-umich/install_v100_dependencies.sh + source ${HOME}/sw/greatlakes/v100/venvs/warpx-v100/bin/activate + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/install_v100_dependencies.sh + :language: bash + + + .. tab-item:: A100 Nodes + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + .. note:: + + This section is TODO. + + +.. _building-greatlakes-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + + .. tab-item:: V100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100 + + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100 -j 8 + + The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. + Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100_py + + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100_py -j 8 --target pip_install + + + .. tab-item:: A100 Nodes + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + .. note:: + + This section is TODO. + +Now, you can :ref:`submit Great Lakes compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit greatlakes jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch``. + + +.. _building-greatlakes-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the greatlakes_v100_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. + + +.. _running-cpp-greatlakes: + +Running +------- + +.. tab-set:: + + .. tab-item:: V100 (16GB) GPUs + + The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly) on the supercomputer Great Lakes at University of Michigan. + This partition has `20 nodes, each with two V100 GPUs `__. + + Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. + Note that we run one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/greatlakes_v100.sbatch + :language: bash + :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch``. + + To run a simulation, copy the lines above to a file ``greatlakes_v100.sbatch`` and run + + .. code-block:: bash + + sbatch greatlakes_v100.sbatch + + to submit the job. + + + .. tab-item:: A100 (80GB) GPUs + + This partition has `2 nodes, each with four A100 GPUs `__ that provide 80 GB HBM per A100 GPU. + To the user, each node will appear as if it has 8 A100 GPUs with 40 GB memory each. + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + The Great Lakes CPU partition as up to `455 nodes `__, each with 2x Intel Xeon Gold 6154 CPUs and 180 GB RAM. + + .. note:: + + This section is TODO. + + +.. _post-processing-greatlakes: + +Post-Processing +--------------- + +For post-processing, many users prefer to use the online `Jupyter service `__ (`documentation `__) that is directly connected to the cluster's fast filesystem. + +.. note:: + + This section is a stub and contributions are welcome. + We can document further details, e.g., which recommended post-processing Python software to install or how to customize Jupyter kernels here. diff --git a/Docs/source/install/hpc/lumi.rst b/Docs/source/install/hpc/lumi.rst index ee65d5b421b..2bae4d21599 100644 --- a/Docs/source/install/hpc/lumi.rst +++ b/Docs/source/install/hpc/lumi.rst @@ -3,7 +3,7 @@ LUMI (CSC) ========== -The `LUMI cluster `_ is located at CSC (Finland). +The `LUMI cluster `__ is located at CSC (Finland). Each node contains 4 AMD MI250X GPUs, each with 2 Graphics Compute Dies (GCDs) for a total of 8 GCDs per node. You can think of the 8 GCDs as 8 separate GPUs, each having 64 GB of high-bandwidth memory (HBM2E). @@ -12,8 +12,11 @@ Introduction If you are new to this system, **please see the following resources**: -* `Lumi user guide `_ -* Batch system: `Slurm `_ +* `Lumi user guide `__ + + * `Project Maintainance `__ and `SSH Key management `__ + * `Quotas and projects `__ +* Batch system: `Slurm `__ * `Data analytics and visualization `__ * `Production directories `__: @@ -48,9 +51,16 @@ Create it now: .. literalinclude:: ../../../../Tools/machines/lumi-csc/lumi_warpx.profile.example :language: bash -Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor -such as ``nano``, ``emacs``, or ``vim`` (all available by default on -LUMI login nodes). +Edit the 2nd line of this script, which sets the ``export proj="project_..."`` variable using a text editor +such as ``nano``, ``emacs``, or ``vim`` (all available by default on LUMI login nodes). +You can find out your project name by running ``lumi-ldap-userinfo`` on LUMI. +For example, if you are member of the project ``project_465000559``, then run ``nano $HOME/lumi_impactx.profile`` and edit line 2 to read: + +.. code-block:: bash + + export proj="project_465000559" + +Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). .. important:: @@ -88,7 +98,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_lumi - cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_lumi/bin/``. @@ -98,7 +108,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lumi_py - cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi_py -j 16 --target pip_install @@ -143,11 +153,26 @@ Running MI250X GPUs (2x64 GB) ^^^^^^^^^^^^^^^^^^^^^ -In non-interactive runs: +The GPU partition on the supercomputer LUMI at CSC has up to `2978 nodes `__, each with 8 Graphics Compute Dies (GCDs). +WarpX runs one MPI rank per Graphics Compute Die. + +For interactive runs, simply use the aliases ``getNode`` or ``runNode ...``. -.. literalinclude:: ../../../../Tools/machines/lumi-csc/submit.sh +The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly). +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` or the concete inputs file. +Copy the executable or point to it via ``EXE`` and adjust the path for the ``INPUTS`` variable accordingly. + +.. literalinclude:: ../../../../Tools/machines/lumi-csc/lumi.sbatch :language: bash - :caption: You can copy this file from ``Tools/machines/lumi-csc/submit.sh``. + :caption: You can copy this file from ``Tools/machines/lumi-csc/lumi.sbatch``. + +To run a simulation, copy the lines above to a file ``lumi.sbatch`` and run + +.. code-block:: bash + + sbatch lumi.sbatch + +to submit the job. .. _post-processing-lumi: diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index e9ebcd4e1de..14972566abc 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -76,7 +76,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx/bin/activate + source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate .. dropdown:: Script Details :color: light @@ -126,7 +126,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx/bin/activate + source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate .. dropdown:: Script Details :color: light diff --git a/Docs/source/install/users.rst b/Docs/source/install/users.rst index b7893e248c0..e56d2d8ac43 100644 --- a/Docs/source/install/users.rst +++ b/Docs/source/install/users.rst @@ -45,12 +45,12 @@ A package for WarpX is available via the `Conda `_ package man .. tip:: - We recommend to configure your conda to use the faster `libmamba` `dependency solver `__. + We recommend to configure your conda to use the faster ``libmamba`` `dependency solver `__. .. code-block:: bash - conda update -n base conda - conda install -n base conda-libmamba-solver + conda update -y -n base conda + conda install -y -n base conda-libmamba-solver conda config --set solver libmamba We recommend to deactivate that conda self-activates its ``base`` environment. diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 895a6c1392b..84b34b17351 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -207,13 +207,15 @@ @article{Roedel2010 @misc{SandbergPASC24, address = {Zuerich, Switzerland}, -author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Andrew Myers and Ji Qiang and Jean-Luc Vay and Axel Huebl}, booktitle = {Proc. of PASC24}, -note = {submitted}, +note = {accepted}, series = {PASC'24 - Platform for Advanced Scientific Computing}, title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, venue = {Zuerich, Switzerland}, -year = {2024} +year = {2024}, +doi = {10.48550/arXiv.2402.17248}, +url = {https://arxiv.org/abs/2402.17248} } @inproceedings{SandbergIPAC23, @@ -391,3 +393,51 @@ @article{GrismayerNJP2021 volume = {23}, year = {2021} } + +@article{QiangPhysRevSTAB2006, + title = {Three-dimensional quasistatic model for high brightness beam dynamics simulation}, + author = {Qiang, Ji and Lidia, Steve and Ryne, Robert D. and Limborg-Deprey, Cecile}, + journal = {Phys. Rev. ST Accel. Beams}, + volume = {9}, + issue = {4}, + pages = {044204}, + numpages = {10}, + year = {2006}, + month = {Apr}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevSTAB.9.044204} +} + +@article{QiangPhysRevSTAB2006err, + title = {Erratum: Three-dimensional quasistatic model for high brightness beam dynamics simulation [Phys. Rev. ST Accel. Beams 9, 044204 (2006)]}, + author = {Qiang, Ji and Lidia, Steve and Ryne, Robert D. and Limborg-Deprey, Cecile}, + journal = {Phys. Rev. ST Accel. Beams}, + volume = {10}, + issue = {12}, + pages = {129901}, + numpages = {2}, + year = {2007}, + month = {Dec}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevSTAB.10.129901} +} + +@book{Wiedemann2015, + author = {Wiedemann, H.}, + isbn = {978-3-319-18317-6}, + publisher = {Springer Cham}, + title = {{Particle Accelerator Physics}}, + doi = {10.1007/978-3-319-18317-6}, + year = {2015} +} + +@article{Vranic2015, + title = {Particle merging algorithm for PIC codes}, + author = {M. Vranic and T. Grismayer and J.L. Martins and R.A. Fonseca and L.O. Silva}, + journal = {Computer Physics Communications}, + volume = {191}, + pages = {65-73}, + year = {2015}, + issn = {0010-4655}, + doi = {https://doi.org/10.1016/j.cpc.2015.01.020}, +} diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 3fb8320c531..b4f494d8382 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -73,13 +73,13 @@ If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above en_e\vec{E} = -\vec{J}_e\times\vec{B}-\nabla\cdot{\overleftrightarrow P}_e+\vec{R}_e. Making the further simplifying assumptions that the electron pressure is isotropic and that -the electron drag term can be written as a simple resistance -i.e. :math:`\vec{R}_e = en_e\vec{\eta}\cdot\vec{J}`, brings us to the implemented form of +the electron drag term can be written using a simple resistivity (:math:`\eta`) and hyper-resistivity (:math:`\eta_h`) +i.e. :math:`\vec{R}_e = en_e(\eta-\eta_h \nabla^2)\vec{J}`, brings us to the implemented form of Ohm's law: .. math:: - \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\vec{\eta}\cdot\vec{J}. + \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\eta\vec{J}-\eta_h \nabla^2\vec{J}. Lastly, if an electron temperature is given from which the electron pressure can be calculated, the model is fully constrained and can be evolved given initial @@ -167,5 +167,13 @@ input parameters, :math:`T_{e0}`, :math:`n_0` and :math:`\gamma` using The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. +Electron current +^^^^^^^^^^^^^^^^ + +WarpX's displacement current diagnostic can be used to output the electron current in +the kinetic-fluid hybrid model since in the absence of kinetic electrons, and under +the assumption of zero displacement current, that diagnostic simply calculates the +hybrid model's electron current. + .. bibliography:: :keyprefix: kfhm- diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 0c5fe85973f..40c16a06df5 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -157,16 +157,28 @@ Overall simulation parameters where :math:`\boldsymbol{\beta}` is the average (normalized) velocity of the considered species (which can be relativistic). See, e.g., :cite:t:`param-Vaypop2008` for more information. - See the `AMReX documentation `_ - for details of the MLMG solver (the default solver used with electrostatic - simulations). The default behavior of the code is to check whether there is - non-zero charge density in the system and if so force the MLMG solver to - use the solution max norm when checking convergence. If there is no charge - density, the MLMG solver will switch to using the initial guess max norm - error when evaluating convergence and an absolute error tolerance of - :math:`10^{-6}` :math:`\mathrm{V/m}^2` will be used (unless a different - non-zero value is specified by the user via - ``warpx.self_fields_absolute_tolerance``). +* ``warpx.poisson_solver`` (`string`) optional (default `multigrid`) + + * ``multigrid``: Poisson's equation is solved using an iterative multigrid (MLMG) solver. + See the `AMReX documentation `__ + for details of the MLMG solver (the default solver used with electrostatic + simulations). The default behavior of the code is to check whether there is + non-zero charge density in the system and if so force the MLMG solver to + use the solution max norm when checking convergence. If there is no charge + density, the MLMG solver will switch to using the initial guess max norm + error when evaluating convergence and an absolute error tolerance of + :math:`10^{-6}` :math:`\mathrm{V/m}^2` will be used (unless a different + non-zero value is specified by the user via + ``warpx.self_fields_absolute_tolerance``). + + * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). + See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. + It only works in 3D and it requires the compilation flag ``-DWarpX_PSATD=ON``. + If mesh refinement is enabled, this solver only works on the coarsest level. + On the refined patches, the Poisson equation is solved with the multigrid solver. + In electrostatic mode, this solver requires open field boundary conditions (``boundary.field_lo,hi = open``). + In electromagnetic mode, this solver can be used to initialize the species' self fields + (``.initialize_self_fields=1``) provided that the field BCs are PML (``boundary.field_lo,hi = PML``). * ``warpx.self_fields_required_precision`` (`float`, default: 1.e-11) The relative precision with which the electrostatic space-charge fields should @@ -395,11 +407,19 @@ Domain Boundary Conditions * ``damped``: This is the recommended option in the moving direction when using the spectral solver with moving window (currently only supported along z). This boundary condition applies a damping factor to the electric and magnetic fields in the outer half of the guard cells, using a sine squared profile. As the spectral solver is by nature periodic, the damping prevents fields from wrapping around to the other end of the domain when the periodicity is not desired. This boundary condition is only valid when using the spectral solver. * ``pec``: This option can be used to set a Perfect Electric Conductor at the simulation boundary. Please see the :ref:`PEC theory section ` for more details. Note that PEC boundary is invalid at `r=0` for the RZ solver. Please use ``none`` option. This boundary condition does not work with the spectral solver. - If an electrostatic field solve is used the boundary potentials can also be set through ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`). * ``none``: No boundary condition is applied to the fields with the electromagnetic solver. This option must be used for the RZ-solver at `r=0`. - * ``neumann``: For the electrostatic solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. + * ``neumann``: For the electrostatic multigrid solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. + + * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method. + +* ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) + Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers + (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential + in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), + setting these variables will trigger an electrostatic solve at ``t=0``, to compute the initial + electric field produced by the boundaries. * ``boundary.particle_lo`` and ``boundary.particle_hi`` (`2 strings` for 2D, `3 strings` for 3D, `absorbing` by default) Options are: @@ -411,6 +431,12 @@ Domain Boundary Conditions * ``Reflecting``: Particles leaving the boundary are reflected from the boundary back into the domain. When ``boundary.reflect_all_velocities`` is false, the sign of only the normal velocity is changed, otherwise the sign of all velocities are changed. + * ``Thermal``: Particles leaving the boundary are reflected from the boundary back into the domain + and their velocities are thermalized. The tangential velocity components are sampled from ``gaussian`` distribution + and the component normal to the boundary is sampled from ``gaussian flux`` distribution. + The standard deviation for these distributions should be provided for each species using + ``boundary..u_th``. The same standard deviation is used to sample all components. + * ``boundary.reflect_all_velocities`` (`bool`) optional (default `false`) For a reflecting boundary condition, this flags whether the sign of only the normal velocity is changed or all velocities. @@ -473,9 +499,12 @@ Embedded Boundary Conditions the interior of the embeddded boundary is where the function value is positive. * ``warpx.eb_potential(x,y,z,t)`` (`string`) - Only used when ``warpx.do_electrostatic=labframe``. Gives the value of - the electric potential at the surface of the embedded boundary, - as a function of `x`, `y`, `z` and time. This function is also evaluated + Gives the value of the electric potential at the surface of the embedded boundary, + as a function of `x`, `y`, `z` and `t`. With electrostatic solvers (i.e., with + ``warpx.do_electrostatic = ...``), this is used in order to compute the potential + in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), + setting this variable will trigger an electrostatic solve at ``t=0``, to compute the initial + electric field produced by the boundaries. Note that this function is also evaluated inside the embedded boundary. For this reason, it is important to define this function in such a way that it is constant inside the embedded boundary. @@ -548,7 +577,7 @@ Distribution across MPI ranks and parallelization For example, if there are 4 boxes per rank and `load_balance_knapsack_factor=2`, no more than 8 boxes can be assigned to any rank. -* ``algo.load_balance_costs_update`` (`heuristic` or `timers` or `gpuclock`) optional (default `timers`) +* ``algo.load_balance_costs_update`` (``heuristic`` or ``timers``) optional (default ``timers``) If this is `heuristic`: load balance costs are updated according to a measure of particles and cells assigned to each box of the domain. The cost :math:`c` is computed as @@ -565,10 +594,6 @@ Distribution across MPI ranks and parallelization If this is `timers`: costs are updated according to in-code timers. - If this is `gpuclock`: [**requires to compile with option** ``-DWarpX_GPUCLOCK=ON``] - costs are measured as (max-over-threads) time spent in current deposition - routine (only applies when running on GPUs). - * ``algo.costs_heuristic_particles_wt`` (`float`) optional Particle weight factor used in `Heuristic` strategy for costs update; if running on GPU, the particle weight is set to a value determined from single-GPU tests on Summit, @@ -799,7 +824,20 @@ Particle initialization * ``.focal_distance`` (optional, distance between the beam centroid and the position of the focal plane of the beam, along the direction of the beam mean velocity; space charge is ignored in the initialization of the particles) - If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the size of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the sizes of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + + Usually, in accelerator physics the operative quantities are the normalized emittances :math:`\epsilon_{x,y}` and beta functions :math:`\beta_{x,y}`. + We assume that the beam travels along :math:`z` and we mark the quantities evaluated at the focal plane with a :math:`*`. + Therefore, the normalized transverse emittances and beta functions are related to the focal distance :math:`f = z - z^*`, the beam sizes :math:`\sigma_{x,y}` (which in the code are ``x_rms``, ``y_rms``), the beam relativistic Lorentz factor :math:`\gamma`, and the normalized momentum spread :math:`\Delta u_{x,y}` according to the equations below (:cite:t:`param-Wiedemann2015`). + + .. math:: + + \Delta u_{x,y} &= \frac{\epsilon^*_{x,y}}{\sigma^*_{x,y}}, + + \sigma*_{x, y} &= \sqrt{ \frac{ \epsilon^*_{x,y} \beta^*_{x,y} }{\gamma}}, + + \sigma_{x,y}(z) &= \sigma^*_{x,y} \sqrt{1 + \left( \frac{z - z^*}{\beta^*_{x,y}} \right)^2} + * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: @@ -1231,25 +1269,39 @@ Particle initialization * ``.do_resampling`` (`0` or `1`) optional (default `0`) If `1` resampling is performed for this species. This means that the number of macroparticles will be reduced at specific timesteps while preserving the distribution function as much as - possible (in particular the weight of the remaining particles will be increased on average). + possible (details depend on the chosen resampling algorithm). This can be useful in situations with continuous creation of particles (e.g. with ionization or with QED effects). At least one resampling trigger (see below) must be specified to actually perform resampling. * ``.resampling_algorithm`` (`string`) optional (default `leveling_thinning`) - The algorithm used for resampling. Currently there is only one option, which is already set by - default: + The algorithm used for resampling: * ``leveling_thinning`` This algorithm is defined in :cite:t:`param-MuravievCPC2021`. - It has two parameters: + It has one parameter: * ``.resampling_algorithm_target_ratio`` (`float`) optional (default `1.5`) This **roughly** corresponds to the ratio between the number of particles before and after resampling. - * ``.resampling_algorithm_min_ppc`` (`int`) optional (default `1`) - Resampling is not performed in cells with a number of macroparticles strictly smaller - than this parameter. + * ``velocity_coincidence_thinning``` The particles are sorted into phase space + cells and merged, similar to the approach described in :cite:t:`param-Vranic2015`. + It has three parameters: + + * ``.resampling_algorithm_delta_ur`` (`float`) + The width of momentum cells used in clustering particles, in m/s. + + * ``.resampling_algorithm_n_theta`` (`int`) + The number of cell divisions to use in the :math:`\theta` direction + when clustering the particle velocities. + + * ``.resampling_algorithm_n_phi`` (`int`) + The number of cell divisions to use in the :math:`\phi` direction + when clustering the particle velocities. + +* ``.resampling_min_ppc`` (`int`) optional (default `1`) + Resampling is not performed in cells with a number of macroparticles strictly smaller + than this parameter. * ``.resampling_trigger_intervals`` (`string`) optional (default `0`) Using the `Intervals parser`_ syntax, this string defines timesteps at which resampling is @@ -2254,6 +2306,9 @@ Maxwell solver: kinetic-fluid hybrid * ``hybrid_pic_model.plasma_resistivity(rho,J)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. +* ``hybrid_pic_model.plasma_hyper_resistivity`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma hyper-resistivity in :math:`\Omega m^3`. + * ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the external current (on the grid) in :math:`A/m^2`. @@ -2503,18 +2558,18 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. * ``.openpmd_encoding`` (optional, ``v`` (variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. - openPMD `file output encoding `__. + openPMD `file output encoding `__. File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). - ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. + ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. Default: ``f`` (full diagnostics) * ``.adios2_operator.type`` (``zfp``, ``blosc``) optional, - `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. * ``.adios2_operator.parameters.*`` optional, - `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. - A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): + A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): .. code-block:: text @@ -2533,11 +2588,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a .adios2_operator.parameters.precision = 3 * ``.adios2_engine.type`` (``bp4``, ``sst``, ``ssc``, ``dataman``) optional, - `ADIOS2 Engine type `__ for `openPMD `_ data dumps. + `ADIOS2 Engine type `__ for `openPMD `_ data dumps. See full list of engines at `ADIOS2 readthedocs `__ * ``.adios2_engine.parameters.*`` optional, - `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. + `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. An example for parameters for the BP engine are setting the number of writers (``NumAggregators``), transparently redirecting data to burst buffers etc. A detailed list of engine-specific parameters are available at the official `ADIOS2 documentation `__ @@ -2634,9 +2689,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``..variables`` (list of `strings` separated by spaces, optional) List of particle quantities to write to output. - Choices are ``w`` for the particle weight and ``ux`` ``uy`` ``uz`` for the particle momenta. - By default, all particle quantities are written. - If ``..variables = none``, no particle data are written, except for particle positions, which are always included. + Choices are ``x``, ``y``, ``z`` for the particle positions (3D and RZ), ``x`` & ``z`` in 2D, ``z`` in 1D, + ``w`` for the particle weight and ``ux``, ``uy``, ``uz`` for the particle momenta. + When using the lab-frame electrostatic solver, ``phi`` (electrostatic potential, on the macroparticles) is also available. + By default, all particle quantities (except ``phi``) are written. + If ``..variables = none``, no particle data are written. * ``..random_fraction`` (`float`) optional If provided ``..random_fraction = a``, only `a` fraction of the particle data of this species will be dumped randomly in diag ````, i.e. if `rand() < a`, this particle will be dumped, where `rand()` denotes a random number generator. @@ -3310,6 +3367,8 @@ Lookup tables store pre-computed values for functions used by the QED modules. * ``qed_bw.save_table_in`` (`string`): where to save the lookup table + Alternatively, the lookup table can be generated using a standalone tool (see :ref:`qed tools section `). + * ``load``: a lookup table is loaded from a pre-generated binary file. The following parameter must be specified: @@ -3347,6 +3406,8 @@ Lookup tables store pre-computed values for functions used by the QED modules. * ``qed_qs.save_table_in`` (`string`): where to save the lookup table + Alternatively, the lookup table can be generated using a standalone tool (see :ref:`qed tools section `). + * ``load``: a lookup table is loaded from a pre-generated binary file. The following parameter must be specified: diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 639b57d219e..3aba5f165de 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -35,7 +35,7 @@ Simulation and Grid Setup ------------------------- .. autoclass:: pywarpx.picmi.Simulation - :members: step, add_species, add_laser, write_input_file + :members: step, add_species, add_laser, add_applied_field, write_input_file .. autoclass:: pywarpx.picmi.Cartesian3DGrid @@ -69,6 +69,8 @@ which can be used directly inside any PICMI script. The values are in SI units. Applied fields -------------- +Instances of the classes below need to be passed to the method `add_applied_field` of the `Simulation` class. + .. autoclass:: pywarpx.picmi.AnalyticInitialField .. autoclass:: pywarpx.picmi.ConstantAppliedField diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index e68ec9391be..5c5329e18b8 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -12,8 +12,9 @@ This section collects typical user workflows and best practices for WarpX. workflows/domain_decomposition workflows/plot_distribution_mapping workflows/debugging - workflows/libensemble + workflows/generate_lookup_tables_with_tools workflows/plot_timestep_duration workflows/psatd_stencil workflows/archiving workflows/ml_dataset_training + workflows/optimas diff --git a/Docs/source/usage/workflows/domain_decomposition.rst b/Docs/source/usage/workflows/domain_decomposition.rst index 1340ecc10d9..201f2dcbbc5 100644 --- a/Docs/source/usage/workflows/domain_decomposition.rst +++ b/Docs/source/usage/workflows/domain_decomposition.rst @@ -5,6 +5,9 @@ Domain Decomposition WarpX relies on a spatial domain decomposition for MPI parallelization. It provides two different ways for users to specify this decomposition, a `simple` way recommended for most users, and a `flexible` way recommended if more control is desired. The `flexible` method is required for dynamic load balancing to be useful. + +.. _usage_domain_decomposition-simple: + 1. Simple Method ---------------- @@ -14,6 +17,9 @@ The first and simplest method is to provide the ``warpx.numprocs = nx ny nz`` pa If ``warpx.numprocs`` is *not* specified, WarpX will fall back on using the ``amr.max_grid_size`` and ``amr.blocking_factor`` parameters, described below. + +.. _usage_domain_decomposition-general: + 2. More General Method ---------------------- diff --git a/Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst b/Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst new file mode 100644 index 00000000000..7417b7a1f96 --- /dev/null +++ b/Docs/source/usage/workflows/generate_lookup_tables_with_tools.rst @@ -0,0 +1,45 @@ +.. _generate-lookup-tables-with-tools: + +Generate QED lookup tables using the standalone tool +==================================================== + +We provide tools to generate and convert into a human-readable format the QED lookup tables. +Such tools can be compiled with ``cmake`` by setting the flag ``WarpX_QED_TOOLS=ON`` (this +requires both ``PICSAR`` and ``Boost`` libraries). The tools are compiled alongside the WarpX executable +in the folder ``bin``. We report here the help message displayed by the tools: + +.. code-block:: console + + $ ./qed_table_reader -h + ### QED Table Reader ### + Command line options: + -h [NO ARG] Prints all command line arguments + -i [STRING] Name of the file to open + --table [STRING] Either BW (Breit-Wheeler) or QS (Quantum Synchrotron) + --mode [STRING] Precision of the calculations: either DP (double) or SP (single) + -o [STRING] filename to save the lookup table in human-readable format + + $ ./qed_table_generator -h + ### QED Table Generator ### + Command line options: + -h [NO ARG] Prints all command line arguments + --table [STRING] Either BW (Breit-Wheeler) or QS (Quantum Synchrotron) + --mode [STRING] Precision of the calculations: either DP (double) or SP (single) + --dndt_chi_min [DOUBLE] minimum chi parameter for the dNdt table + --dndt_chi_max [DOUBLE] maximum chi parameter for the dNdt table + --dndt_how_many [INTEGR] number of points in the dNdt table + --pair_chi_min [DOUBLE] minimum chi for the pair production table (BW only) + --pair_chi_max [DOUBLE] maximum chi for the pair production table (BW only) + --pair_chi_how_many [INTEGR] number of chi points in the pair production table (BW only) + --pair_frac_how_many [INTEGR] number of frac points in the pair production table (BW only) + --em_chi_min [DOUBLE] minimum chi for the photon emission table (QS only) + --em_chi_max [DOUBLE] maximum chi for the photon emission production table (QS only) + --em_frac_min [DOUBLE] minimum frac for the photon emission production table (QS only) + --em_chi_how_many [INTEGR] number of chi points in the photon emission table (QS only) + --em_frac_how_many [INTEGR] number of frac points in the photon emission table (QS only) + -o [STRING] filename to save the lookup table + +These tools are meant to be compatible with WarpX: ``qed_table_generator`` should generate +tables that can be loaded into WarpX and ``qed_table_reader`` should be able to read tables generated with WarpX. +It is not safe to use these tools to generate a table on a machine using a different endianness with respect to +the machine where the table is used. diff --git a/Docs/source/usage/workflows/libensemble.rst b/Docs/source/usage/workflows/libensemble.rst deleted file mode 100644 index af42ae6adf9..00000000000 --- a/Docs/source/usage/workflows/libensemble.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _libensemble: - -Run LibEnsemble on WarpX -======================== - -`LibEnsemble `__ is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. -While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. -If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., LibEnsemble provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (LibEnsemble targets modern HPC platforms like Summit). - -Scripts to run LibEnsemble on WarpX simulations can be found in ``WarpX/Tools/LibEnsemble/``. -This documentation does not aim at giving a training on LibEnsemble, so please refer to the `LibEnsemble documentation `__ for technical details. - -WarpX example problem for LibEnsemble study -------------------------------------------- - -The WarpX example is built on a 2D input file (so that 1 simulation take < 1 min) of a 2-stage laser-wakefield simulation in a boosted frame. -It aims at optimizing emittance preservation in the coupling between two consecutive plasma accelerator stages. -Each stage accelerates an externally-injected electron beam, which charge has been tuned to show a decent amount of Ez field flattening due to longitudinal beam loading. -Each stage has a parabolic transverse profile to guide the laser pulse, and a uniform longitudinal profile with cos-shape ramps and the entrance and at the exit. -A fresh laser pulse is introduced at the entrance of each stage and deleted at the exit. -The beam transverse distribution is matched to the first stage and the beam is injected at the beginning of the plateau of the first stage, so that the emittance is conserved in the first stage. -The two stages are separated by a few-cm gap, and a focusing lens is placed in-between. -Note that this is a very low resolution simulation to show an example, so it is **not** close to numerical convergence. - -In this example, we allow LibEnsemble to tune four **input parameters**: - - - Length of the downramp of the first stage - - - Longitudinal position of the focusing lens (between the two stages) - - - Strength of the focusing lens - - - Length of the downramp of the second stage - -The **output parameter** that LibEnsemble minimizes is the beam emittance at the exit of the second stage, while making sure the charge loss is small. - -The scripts provided can run on a local machine or on the Summit supercomputer at OLCF. -Two options are available: random sampling of parameter space or optimization on the output parameter. -For the latter, we are using the Asynchronously Parallel Optimization Solver for finding Multiple Minima `APOSMM `__ method provided by LibEnsemble. - -Install LibEnsemble -------------------- - -Besides a working WarpX executable, you have to install libEnsemble and its dependencies. - -You can either install all packages via `conda` (recommended), - -.. code-block:: sh - - conda install -c conda-forge libensemble matplotlib numpy scipy yt - -or try to install the same dependencies via `pip` (pick one *or* the other; note our :ref:`installation details on Summit `): - -.. literalinclude:: ../../../../Tools/LibEnsemble/requirements.txt - - -What's in ``Tools/LibEnsemble``? --------------------------------- - -See the `LibEnsemble User Guide `__ for an overview of LibEnsemble concepts. -In a nutshell, a user needs to define - - - A generator function ``gen_f`` that will generate inputs for the simulation, which can be done uniformly, randomly or using an optimizer. - The generator output, i.e., the simulation input, is called ``'x'``. - The generator is provided by LibEnsemble. - When the generator is an optimizer, it takes the simulation output called ``'f'`` as an input. - - - A simulation function ``sim_f`` that will take ``'x'`` as an input and return a single output value ``'f'``. - In our example, ``sim_f`` modifies the WarpX input file depending on ``'x'``, launches a WarpX simulation and reads the simulation output plotfile to extract ``'f'``. - - - An allocator function ``alloc_f`` that will feed available workers with tasks. - This is provided by LibEnsemble. - -The files in ``Tools/LibEnsemble/`` are: - -``run_libensemble_on_warpx.py`` - This is the main LibEnsemble script. - It imports ``gen_f`` and ``alloc_f`` from LibEnsemble, ``sim_f`` from file ``warpx_simf.py`` (see below), defines dictionaries for parameters of each of these objects (``gen_specs`` includes lower and upper bound of each element in the input array ``'x'``, ``alloc_specs`` and ``sim_specs`` respectively) and runs LibEnsemble. - -``warpx_simf.py`` - defines the ``sim_f`` function called ``run_warpx``: - - .. doxygenfunction:: run_warpx - -``sim/inputs`` - WarpX input file. Some of its parameters are modified in ``run_warpx``. - -``write_sim_input.py`` - (util) update one WarpX input file depending on values in ``'x'`` for this run. - - .. doxygenfunction:: write_sim_input - -``read_sim_output.py`` - (util) Read WarpX plotfile and return ``'f'``. - - .. doxygenfunction:: read_sim_output - -``plot_results.py`` - (util) Read LibEnsemble output files ``.npy`` and ``.pickle`` and plot output ``'f'`` (and other output, just for convenience) as a function of input from all simulations. - -``all_machine_specs.py`` - (util) dictionaries of machine-specific parameters. - For convenience, the maximum number of WarpX runs is defined there. - -``summit_submit_mproc.sh`` - Submission script for LibEnsemble+WarpX on Summit. - Make sure to edit this file and add your project ID for allocations. - -Run the example ---------------- - -On Summit or for a local run, LibEnsemble can run with a Random Number Generator (easiest, good as a first try) or with an optimizer (requires python package ``nlopt``). -This is set by the variable ``generator_type`` in ``run_libensemble_on_warpx.py``. -We hereafter assume that all Python modules required are installed and that a WarpX executable is available. - -Run locally -^^^^^^^^^^^ - -Adjust the ``local_specs`` dictionary in ``all_machine_specs.py`` to fix the path to the WarpX executable (and optionally change the number of cores and OpenMP threads), and run - -.. code-block:: sh - - python run_libensemble_on_warpx.py --comms local --nworkers 3 - -This is adapted to a 4-core machine, as it will use: - -- 1 process to run LibEnsemble - -- 1 process (among the 3 workers) to run the generator - -- 2 processes to run 2 concurrent simulations - -Run on Summit at OLCF -^^^^^^^^^^^^^^^^^^^^^ - -- ``cp -r $HOME/warpx/Tools/LibEnsemble/* sim_directory`` -- modify ``run_libensemble_on_warpx.py`` to have ``machine = 'summit'`` -- modify ``all_machine_specs.py`` to put the right path to the WarpX executable -- modify ``summit_submit_mproc.sh`` to set ``LIBE_PLOTS`` to ``false`` and set the project ID -- ``bsub summit_submit_mproc.sh``: - -.. code-block:: sh - - bsub summit_submit_mproc.sh diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst index 6e60a318bee..450e0fe2879 100644 --- a/Docs/source/usage/workflows/ml_dataset_training.rst +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -14,42 +14,42 @@ For example, a simulation determined by the following input script .. literalinclude:: ml_materials/run_warpx_training.py :language: python -In this section we walk through a workflow for data processing and model training. +In this section we walk through a workflow for data processing and model training, using data from this input script as an example. +The simulation output is stored in an online `Zenodo archive `__, in the ``lab_particle_diags`` directory. +In the example scripts provided here, the data is downloaded from the Zenodo archive, properly formatted, and used to train a neural network. This workflow was developed and first presented in :cite:t:`ml-SandbergIPAC23,ml-SandbergPASC24`. - -This assumes you have an up-to-date environment with PyTorch and openPMD. +It assumes you have an up-to-date environment with PyTorch and openPMD. Data Cleaning ------------- -It is important to inspect the data for artifacts to +It is important to inspect the data for artifacts, to check that input/output data make sense. -If we plot the final phase space for beams 1-8, -the particle data is distributed in a single blob, -as shown by :numref:`fig_phase_space_beam_1` for beam 1. -This is as we expect and what is optimal for training neural networks. +If we plot the final phase space of the particle beam, +shown in :numref:`fig_unclean_phase_space`. +we see outlying particles. +Looking closer at the z-pz space, we see that some particles were not trapped in the accelerating region of the wake and have much less energy than the rest of the beam. + +.. _fig_unclean_phase_space: -.. _fig_phase_space_beam_1: +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/unclean_stage_0.png + :alt: Plot showing the final phase space projections of a particle beam through a laser-plasma acceleration element where some beam particles were not accelerated. -.. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png - :alt: Plot showing the final phase space projections for beam 1 of the training data, for a surrogate to stage 1. + The final phase space projections of a particle beam through a laser-plasma acceleration element where some beam particles were not accelerated. - The final phase space projections for beam 1 of the training data, for a surrogate to stage 1. +To assist our neural network in learning dynamics of interest, we filter out these particles. +It is sufficient for our purposes to select particles that are not too far back, setting +``particle_selection={'z':[0.280025, None]}``. +After filtering, we can see in :numref:`fig_clean_phase_space` that the beam phase space projections are much cleaner -- this is the beam we want to train on. -.. _fig_phase_space_beam_0: +.. _fig_clean_phase_space: -.. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png - :alt: Plot showing the final phase space projections for beam 0 of the training data, for a surrogate to stage 0. +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/clean_stage_0.png + :alt: Plot showing the final phase space projections of a particle beam through a laser-plasma acceleration element after filtering out outlying particles. - The final phase space projections for beam 0 of the training data, for a surrogate to stage 0 + The final phase space projections of a particle beam through a laser-plasma acceleration element after filtering out outlying particles. -On the other hand, the final phase space for beam 0, shown in :numref:`fig_phase_space_beam_1`, -has a halo of outlying particles. -Looking closer at the z-pz space, we see that some particles got caught in a decelerating -region of the wake, have slipped back and are much slower than the rest of the beam. -To assist our neural network in learning dynamics of interest, we filter out these particles. -It is sufficient for our purposes to select particles that are not too far back, setting -``particle_selection={'z':[0.28002, None]}``. Then a particle tracker is set up to make sure +A particle tracker is set up to make sure we consistently filter out these particles from both the initial and final data. .. literalinclude:: ml_materials/create_dataset.py @@ -58,6 +58,9 @@ we consistently filter out these particles from both the initial and final data. :start-after: # Manual: Particle tracking START :end-before: # Manual: Particle tracking END +This data cleaning ensures that the particle data is distributed in a single blob, +as is optimal for training neural networks. + Create Normalized Dataset ------------------------- @@ -119,7 +122,12 @@ This data are converted to an :math:`N\times 6` numpy array and then to a PyTorc Save Normalizations and Normalized Data ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -With the data properly normalized, it and the normalizations are saved to file for +The data is split into training and testing subsets. +We take most of the data (70%) for training, meaning that data is used to update +the neural network parameters. +The testing data is reserved to determine how well the neural network generalizes; +that is, how well the neural network performs on data that wasn't used to update the neural network parameters. +With the data split and properly normalized, it and the normalizations are saved to file for use in training and inference. .. literalinclude:: ml_materials/create_dataset.py @@ -131,13 +139,22 @@ use in training and inference. Neural Network Structure ------------------------ -It was found in :cite:t:`ml-SandbergPASC24` that reasonable surrogate models are obtained with -shallow feedforward neural networks consisting of fewer than 10 hidden layers and -just under 1000 nodes per layer. +It was found in :cite:t:`ml-SandbergPASC24` that a reasonable surrogate model is obtained with +shallow feedforward neural networks consisting of about 5 hidden layers and 700-900 nodes per layer. The example shown here uses 3 hidden layers and 20 nodes per layer and is trained for 10 epochs. +Some utility functions for creating neural networks are provided in the script below. +These are mostly convenience wrappers and utilities for working with `PyTorch `__ neural network objects. +This script is imported in the training scripts shown later. + +.. dropdown:: Python neural network class definitions + :color: light + :icon: info + :animate: fade-in-slide-down + .. literalinclude:: ml_materials/neural_network_classes.py + :language: python3 Train and Save Neural Network ----------------------------- @@ -188,8 +205,8 @@ which is later divided by the size of the dataset in the training loop. :start-after: # Manual: Test function START :end-before: # Manual: Test function END -Train Loop -^^^^^^^^^^ +Training Loop +^^^^^^^^^^^^^ The full training loop performs ``n_epochs`` number of iterations. At each iteration the training and testing functions are called, @@ -228,14 +245,14 @@ When the test-loss starts to trend flat or even upward, the neural network is no .. _fig_train_test_loss: -.. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/beam_stage_0_training_testing_error.png :alt: Plot of training and testing loss curves versus number of training epochs. Training (in blue) and testing (in green) loss curves versus number of training epochs. .. _fig_train_evaluation: -.. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png +.. figure:: https://gist.githubusercontent.com/RTSandberg/649a81cc0e7926684f103729483eff90/raw/095ac2daccbcf197fa4e18a8f8505711b27e807a/beam_stage_0_model_evaluation.png :alt: Plot comparing model prediction with simulation output. A comparison of model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). @@ -243,7 +260,7 @@ When the test-loss starts to trend flat or even upward, the neural network is no A visual inspection of the model prediction can be seen in :numref:`fig_train_evaluation`. This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. -A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, +A more accurate model is obtained with 5 hidden layers and 900 nodes per layer, as discussed in :cite:t:`ml-SandbergPASC24`. These figures can be generated with the following Python script. @@ -261,7 +278,7 @@ Surrogate Usage in Accelerator Physics ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A neural network such as the one we trained here can be incorporated in other BLAST codes. -`Consider the example using neural networks in ImpactX `__. +Consider this `example using neural network surrogates of WarpX simulations in ImpactX `__. .. bibliography:: :keyprefix: ml- diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py index 08000105479..ecd182c8802 100644 --- a/Docs/source/usage/workflows/ml_materials/create_dataset.py +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -7,17 +7,16 @@ # Authors: Ryan Sandberg # License: BSD-3-Clause-LBNL # + import os -import tarfile +import zipfile from urllib import request import numpy as np - -c = 2.998e8 - -from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker import torch +from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker +c = 2.998e8 ############### def sanitize_dir_strings(*dir_strings): @@ -31,14 +30,15 @@ def sanitize_dir_strings(*dir_strings): return dir_strings def download_and_unzip(url, data_dir): - request.urlretrieve(url, data_dir) - with tarfile.open(data_dir) as tar_dataset: - tar_dataset.extractall() + request.urlretrieve(url, data_dir) + with zipfile.ZipFile(data_dir, 'r') as zip_dataset: + zip_dataset.extractall() def create_source_target_data(data_dir, species, source_index=0, target_index=-1, + survivor_select_index=-1, particle_selection=None ): """Create dataset from openPMD files @@ -67,7 +67,7 @@ def create_source_target_data(data_dir, relevant_times = [ts.t[source_index], ts.t[target_index]] # Manual: Particle tracking START - iteration = ts.iterations[target_index] + iteration = ts.iterations[survivor_select_index] pt = ParticleTracker( ts, species=species, iteration=iteration, @@ -116,7 +116,6 @@ def create_source_target_data(data_dir, return source_data, source_means, source_stds, target_data, target_means, target_stds, relevant_times - def save_warpx_surrogate_data(dataset_fullpath_filename, diag_dir, species, @@ -128,12 +127,14 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, particle_selection=None ): - source_target_data = create_source_target_data(data_dir=diag_dir, - species=species, - source_index=source_index, - target_index=target_index, - particle_selection=particle_selection - ) + source_target_data = create_source_target_data( + data_dir=diag_dir, + species=species, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + particle_selection=particle_selection + ) source_data, source_means, source_stds, target_data, target_means, target_stds, times = source_target_data # Manual: Save dataset START @@ -161,11 +162,10 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, ######## end utility functions ############# ######## start dataset creation ############ -data_url = "https://zenodo.org/records/10368972/files/ml_example_training.tar.gz?download=1" -download_and_unzip(data_url, "training_dataset.tar.gz") +data_url = "https://zenodo.org/records/10810754/files/lab_particle_diags.zip?download=1" +download_and_unzip(data_url, "lab_particle_diags.zip") data_dir = "lab_particle_diags/lab_particle_diags/" - # create data set source_index = 0 @@ -178,7 +178,7 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, # improve stage 0 dataset stage_i = 0 -select = {'z':[0.28002, None]} +select = {'z':[0.280025, None]} species = f'beam_stage_{stage_i}' dataset_filename = f'dataset_{species}.pt' dataset_file = 'datasets/' + dataset_filename @@ -193,7 +193,7 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, particle_selection=select ) -for stage_i in range(1,9): +for stage_i in range(1,15): species = f'beam_stage_{stage_i}' dataset_filename = f'dataset_{species}.pt' dataset_file = 'datasets/' + dataset_filename diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py index 8ee098e3e29..054c1b4cfc0 100644 --- a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -1,11 +1,4 @@ #!/usr/bin/env python3 -# -# Copyright 2022-2023 WarpX contributors -# Authors: WarpX team -# License: BSD-3-Clause-LBNL -# -# -*- coding: utf-8 -*- - import math import numpy as np @@ -22,7 +15,7 @@ # Number of cells dim = '3' nx = ny = 128 -nz = 8832 +nz = 35328 #17664 #8832 if dim == 'rz': nr = nx//2 @@ -34,9 +27,9 @@ # Number of processes for static load balancing # Check with your submit script -num_procs = [1, 1, 16*4] +num_procs = [1, 1, 64*4] if dim == 'rz': - num_procs = [1, 16] + num_procs = [1, 64] # Number of time steps gamma_boost = 60. @@ -74,7 +67,7 @@ # plasma region plasma_rlim = 100.e-6 -N_stage = 9 +N_stage = 15 L_plasma_bulk = 0.28 L_ramp = 1.e-9 L_ramp_up = L_ramp @@ -142,12 +135,12 @@ def get_species_of_accelerator_stage(stage_idx, stage_zmin, stage_zmax, N_beam_particles = int(1e6) beam_centroid_z = -107.e-6 beam_rms_z = 2.e-6 -#beam_gammas = [2000 + 13000 * i_stage for i_stage in range(N_stage)] -beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run +beam_gammas = [1960 + 13246 * i_stage for i_stage in range(N_stage)] +#beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run beams = [] for i_stage in range(N_stage): beam_gamma = beam_gammas[i_stage] - sigma_gamma = 0.10 * beam_gamma + sigma_gamma = 0.06 * beam_gamma gaussian_distribution = picmi.GaussianBunchDistribution( n_physical_particles= abs(beam_charge) / q_e, rms_bunch_size=[2.e-6, 2.e-6, beam_rms_z], diff --git a/Docs/source/usage/workflows/ml_materials/train.py b/Docs/source/usage/workflows/ml_materials/train.py index dc62819061c..4de11b9c99e 100644 --- a/Docs/source/usage/workflows/ml_materials/train.py +++ b/Docs/source/usage/workflows/ml_materials/train.py @@ -20,8 +20,8 @@ stage_i = 0 species = f'beam_stage_{stage_i}' source_index = 0 -target_index = 4 -survivor_select_index = 4 +target_index = 1 +survivor_select_index = 1 data_dim = 6 n_in = data_dim diff --git a/Docs/source/usage/workflows/ml_materials/visualize.py b/Docs/source/usage/workflows/ml_materials/visualize.py index e6da1029b67..920efe29909 100644 --- a/Docs/source/usage/workflows/ml_materials/visualize.py +++ b/Docs/source/usage/workflows/ml_materials/visualize.py @@ -7,11 +7,11 @@ # Authors: Ryan Sandberg # License: BSD-3-Clause-LBNL # -from matplotlib import pyplot as plt import neural_network_classes as mynn import numpy as np import torch import torch.nn.functional as F +from matplotlib import pyplot as plt c = 2.998e8 diff --git a/Docs/source/usage/workflows/optimas.rst b/Docs/source/usage/workflows/optimas.rst new file mode 100644 index 00000000000..58e8143b251 --- /dev/null +++ b/Docs/source/usage/workflows/optimas.rst @@ -0,0 +1,11 @@ +.. _optimas: + +Optimizing with Optimas +======================== + +`optimas `__ is an open-source Python library that enables highly scalable parallel optimization, from a typical laptop to exascale HPC systems. +While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. +If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., optimas provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (optimas targets modern HPC platforms like Perlmutter and Frontier). + +A more detailed description of optimas is provided in the `optimas documentation `__. +In particular, the online optimas documentation provides `an example optimization with optimas that runs WarpX simulations `__. diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 01406b7a0fb..8b58790d2e6 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -184,11 +184,10 @@ def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): self.max_steps = 50 self.diag_steps = 5 self.mcc_subcycling_steps = 2 + self.rng = np.random.default_rng(23094290) else: self.mcc_subcycling_steps = None - - if self.dsmc: - self.rng = np.random.default_rng(23094290) + self.rng = np.random.default_rng() self.ion_density_array = np.zeros(self.nz + 1) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py index c6529dabd82..29b5272d8b1 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py @@ -5,39 +5,39 @@ import numpy as np ref_density = np.array([ - 1.27953969e+14, 2.23553999e+14, 2.55384510e+14, 2.55663110e+14, - 2.55805760e+14, 2.55812087e+14, 2.55813911e+14, 2.55754104e+14, - 2.55929601e+14, 2.56085472e+14, 2.55932867e+14, 2.55828121e+14, - 2.55901711e+14, 2.55985074e+14, 2.56182697e+14, 2.56446847e+14, - 2.56483696e+14, 2.56301187e+14, 2.56245301e+14, 2.56797584e+14, - 2.57257907e+14, 2.57023627e+14, 2.56500876e+14, 2.56106851e+14, - 2.56283546e+14, 2.56723967e+14, 2.56960855e+14, 2.56825486e+14, - 2.56674669e+14, 2.56567191e+14, 2.56310927e+14, 2.56361171e+14, - 2.56692197e+14, 2.56743606e+14, 2.56653108e+14, 2.56883854e+14, - 2.56763228e+14, 2.56343726e+14, 2.56385489e+14, 2.56570110e+14, - 2.56538112e+14, 2.56472179e+14, 2.56322922e+14, 2.56195384e+14, - 2.56474576e+14, 2.56764233e+14, 2.56533016e+14, 2.56257170e+14, - 2.56362463e+14, 2.56363962e+14, 2.56311292e+14, 2.56678788e+14, - 2.57061138e+14, 2.56785892e+14, 2.56406603e+14, 2.56334908e+14, - 2.56120051e+14, 2.56003269e+14, 2.56132187e+14, 2.56329572e+14, - 2.56535713e+14, 2.56708950e+14, 2.56661860e+14, 2.56448986e+14, - 2.56386823e+14, 2.56233660e+14, 2.56137632e+14, 2.56206263e+14, - 2.56364996e+14, 2.56483536e+14, 2.56308741e+14, 2.56447231e+14, - 2.56896301e+14, 2.56691405e+14, 2.56170780e+14, 2.56122216e+14, - 2.56427399e+14, 2.56897558e+14, 2.56928868e+14, 2.56659033e+14, - 2.56749993e+14, 2.56952497e+14, 2.56798907e+14, 2.56377081e+14, - 2.56453057e+14, 2.56796632e+14, 2.56944576e+14, 2.57248469e+14, - 2.57279426e+14, 2.56849516e+14, 2.56601834e+14, 2.56850545e+14, - 2.56953072e+14, 2.56442586e+14, 2.56329006e+14, 2.56790661e+14, - 2.57083582e+14, 2.57075550e+14, 2.56719615e+14, 2.56220486e+14, - 2.56222323e+14, 2.56547365e+14, 2.56499423e+14, 2.56434041e+14, - 2.56378587e+14, 2.56249892e+14, 2.56380492e+14, 2.56504513e+14, - 2.56337631e+14, 2.56204891e+14, 2.56325116e+14, 2.56297798e+14, - 2.56112782e+14, 2.56054218e+14, 2.56320120e+14, 2.56580938e+14, - 2.56446800e+14, 2.56267011e+14, 2.56372853e+14, 2.56617592e+14, - 2.56630745e+14, 2.56615242e+14, 2.56625259e+14, 2.56561320e+14, - 2.56640072e+14, 2.56693273e+14, 2.56613237e+14, 2.24169847e+14, - 1.27683197e+14 + 1.27989677e+14, 2.23601330e+14, 2.55400265e+14, 2.55664972e+14, + 2.55806841e+14, 2.55806052e+14, 2.55815865e+14, 2.55755151e+14, + 2.55920582e+14, 2.56078972e+14, 2.55933056e+14, 2.55840862e+14, + 2.56043990e+14, 2.56247464e+14, 2.56289153e+14, 2.56277109e+14, + 2.56242679e+14, 2.56328850e+14, 2.56373595e+14, 2.56395979e+14, + 2.56451842e+14, 2.56601614e+14, 2.56530735e+14, 2.56113125e+14, + 2.56207966e+14, 2.56474622e+14, 2.56378743e+14, 2.56230373e+14, + 2.56271563e+14, 2.56317460e+14, 2.56137709e+14, 2.56104767e+14, + 2.56244330e+14, 2.56188574e+14, 2.56070283e+14, 2.56212277e+14, + 2.56374214e+14, 2.56309276e+14, 2.56199679e+14, 2.56227549e+14, + 2.56434824e+14, 2.56578649e+14, 2.56418136e+14, 2.56781877e+14, + 2.57464056e+14, 2.57125505e+14, 2.56477914e+14, 2.56355343e+14, + 2.56599651e+14, 2.56955102e+14, 2.56988933e+14, 2.56954167e+14, + 2.56848084e+14, 2.56278229e+14, 2.55969967e+14, 2.56075186e+14, + 2.56011026e+14, 2.56054081e+14, 2.56203573e+14, 2.56131373e+14, + 2.56181410e+14, 2.56460942e+14, 2.56519893e+14, 2.56394874e+14, + 2.56273725e+14, 2.56263570e+14, 2.56463482e+14, 2.56541354e+14, + 2.56467160e+14, 2.56523576e+14, 2.56760359e+14, 2.56899283e+14, + 2.56889333e+14, 2.56984101e+14, 2.57009449e+14, 2.57017095e+14, + 2.57027915e+14, 2.57002409e+14, 2.56938087e+14, 2.56604366e+14, + 2.56392558e+14, 2.56481669e+14, 2.56337220e+14, 2.56107563e+14, + 2.56286144e+14, 2.56451638e+14, 2.56547674e+14, 2.56846908e+14, + 2.56725251e+14, 2.56321829e+14, 2.56387226e+14, 2.56750085e+14, + 2.56775478e+14, 2.56589827e+14, 2.56723109e+14, 2.56698688e+14, + 2.56674607e+14, 2.57128951e+14, 2.57144972e+14, 2.56602827e+14, + 2.56564415e+14, 2.56824494e+14, 2.56459673e+14, 2.56106618e+14, + 2.56442627e+14, 2.57106509e+14, 2.57468066e+14, 2.57155708e+14, + 2.56501993e+14, 2.56062276e+14, 2.56109738e+14, 2.56284272e+14, + 2.56320557e+14, 2.56253594e+14, 2.56155365e+14, 2.56044920e+14, + 2.55994560e+14, 2.56166612e+14, 2.56327633e+14, 2.56390898e+14, + 2.56474891e+14, 2.56669566e+14, 2.56646041e+14, 2.56264156e+14, + 2.56041610e+14, 2.56041551e+14, 2.56088641e+14, 2.23853646e+14, + 1.27580207e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index a7a76be46ad..4c46b9e3f10 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -18,39 +18,39 @@ my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) ref_density = np.array([ - 1.27957355e+14, 2.23554080e+14, 2.55373436e+14, 2.55659492e+14, - 2.55814670e+14, 2.55818418e+14, 2.55811882e+14, 2.55742272e+14, - 2.55912888e+14, 2.56086072e+14, 2.55944486e+14, 2.55830183e+14, - 2.55909337e+14, 2.56008609e+14, 2.56205930e+14, 2.56421940e+14, - 2.56369990e+14, 2.56151020e+14, 2.55925823e+14, 2.55924941e+14, - 2.56067211e+14, 2.56264104e+14, 2.56435035e+14, 2.56543804e+14, - 2.56715146e+14, 2.56639305e+14, 2.56509438e+14, 2.56478881e+14, - 2.56406748e+14, 2.56194832e+14, 2.56126186e+14, 2.56442221e+14, - 2.56603784e+14, 2.56592554e+14, 2.56475838e+14, 2.56304135e+14, - 2.56310993e+14, 2.56298883e+14, 2.56386742e+14, 2.56555670e+14, - 2.56588013e+14, 2.56851444e+14, 2.56928531e+14, 2.56637559e+14, - 2.56678652e+14, 2.56827322e+14, 2.56630197e+14, 2.56295404e+14, - 2.56285079e+14, 2.56558116e+14, 2.56676094e+14, 2.56577780e+14, - 2.56599749e+14, 2.56540500e+14, 2.56292984e+14, 2.56230350e+14, - 2.56363607e+14, 2.56553909e+14, 2.56501054e+14, 2.56249684e+14, - 2.56280268e+14, 2.56558208e+14, 2.56437837e+14, 2.56152650e+14, - 2.56143349e+14, 2.56067330e+14, 2.56020624e+14, 2.56039223e+14, - 2.56306096e+14, 2.56693084e+14, 2.56649778e+14, 2.56589778e+14, - 2.56594097e+14, 2.56368788e+14, 2.56290090e+14, 2.56420940e+14, - 2.56581419e+14, 2.56642649e+14, 2.56426887e+14, 2.56360122e+14, - 2.56573424e+14, 2.56679138e+14, 2.56488767e+14, 2.56217444e+14, - 2.56353118e+14, 2.56640765e+14, 2.56809490e+14, 2.56933226e+14, - 2.56633538e+14, 2.56203430e+14, 2.56202958e+14, 2.56564020e+14, - 2.56816347e+14, 2.56709830e+14, 2.56557382e+14, 2.56573904e+14, - 2.56745541e+14, 2.56784430e+14, 2.56580054e+14, 2.56210130e+14, - 2.56271415e+14, 2.56821160e+14, 2.56703292e+14, 2.56169296e+14, - 2.56166549e+14, 2.56467777e+14, 2.56573240e+14, 2.56437594e+14, - 2.56253730e+14, 2.56176123e+14, 2.56351125e+14, 2.56569916e+14, - 2.56761101e+14, 2.56891411e+14, 2.56628312e+14, 2.56180062e+14, - 2.56063564e+14, 2.56189728e+14, 2.56609454e+14, 2.57263643e+14, - 2.57097673e+14, 2.56666761e+14, 2.56622585e+14, 2.56432378e+14, - 2.56386718e+14, 2.56734491e+14, 2.57042448e+14, 2.24471147e+14, - 1.27720853e+14 + 1.27943881e+14, 2.23583097e+14, 2.55396716e+14, 2.55673406e+14, + 2.55827566e+14, 2.55803446e+14, 2.55798707e+14, 2.55748961e+14, + 2.55906413e+14, 2.56063991e+14, 2.55937018e+14, 2.55841390e+14, + 2.55917724e+14, 2.55988641e+14, 2.56052050e+14, 2.56285151e+14, + 2.56647960e+14, 2.56756264e+14, 2.56430158e+14, 2.56117493e+14, + 2.56065302e+14, 2.56265220e+14, 2.56328575e+14, 2.56031495e+14, + 2.56123757e+14, 2.56431173e+14, 2.56385320e+14, 2.56391170e+14, + 2.56561177e+14, 2.56513926e+14, 2.56332201e+14, 2.56252442e+14, + 2.56238982e+14, 2.56216498e+14, 2.56461281e+14, 2.56863199e+14, + 2.56908100e+14, 2.56926112e+14, 2.57001641e+14, 2.56735963e+14, + 2.56315358e+14, 2.56137028e+14, 2.56101418e+14, 2.56276827e+14, + 2.56425668e+14, 2.56181798e+14, 2.56044925e+14, 2.56330387e+14, + 2.56623150e+14, 2.56445316e+14, 2.56292750e+14, 2.56440918e+14, + 2.56433406e+14, 2.56186982e+14, 2.56236390e+14, 2.56469557e+14, + 2.56349704e+14, 2.56487457e+14, 2.56771823e+14, 2.56614683e+14, + 2.56552210e+14, 2.56850291e+14, 2.56783396e+14, 2.56483187e+14, + 2.56510868e+14, 2.56490408e+14, 2.56656042e+14, 2.56820924e+14, + 2.56640314e+14, 2.56465063e+14, 2.56510264e+14, 2.56917331e+14, + 2.57228490e+14, 2.56960593e+14, 2.56587911e+14, 2.56672682e+14, + 2.56774414e+14, 2.56548335e+14, 2.56225540e+14, 2.56079693e+14, + 2.56062796e+14, 2.56054612e+14, 2.56028683e+14, 2.56068820e+14, + 2.56380975e+14, 2.56654914e+14, 2.56776792e+14, 2.56983661e+14, + 2.56989477e+14, 2.56646250e+14, 2.56589639e+14, 2.56946205e+14, + 2.57091201e+14, 2.56913590e+14, 2.56513535e+14, 2.56122597e+14, + 2.56176340e+14, 2.56808001e+14, 2.57239393e+14, 2.56845066e+14, + 2.56662482e+14, 2.56862583e+14, 2.56518922e+14, 2.56155531e+14, + 2.56362794e+14, 2.57203564e+14, 2.57737938e+14, 2.57252026e+14, + 2.56859277e+14, 2.56658995e+14, 2.56357364e+14, 2.56393454e+14, + 2.56714308e+14, 2.57042200e+14, 2.57551087e+14, 2.57502490e+14, + 2.56641118e+14, 2.56401115e+14, 2.56644629e+14, 2.56673096e+14, + 2.56534659e+14, 2.56357745e+14, 2.56455309e+14, 2.56586850e+14, + 2.56442415e+14, 2.56335971e+14, 2.56411429e+14, 2.24109018e+14, + 1.27678869e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_rz b/Examples/Physics_applications/laser_acceleration/inputs_rz index 7365523b574..b61ed6f83ec 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_rz +++ b/Examples/Physics_applications/laser_acceleration/inputs_rz @@ -89,5 +89,5 @@ diagnostics.diags_names = diag1 diag1.intervals = 10 diag1.diag_type = Full diag1.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz rho -diag1.electrons.variables = w ux uy uz orig_x orig_z -diag1.beam.variables = w ux uy uz +diag1.electrons.variables = x y z w ux uy uz orig_x orig_z +diag1.beam.variables = x y z w ux uy uz diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index 18a427aa1c7..29862a30518 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -15,12 +15,6 @@ Although laser-ion acceleration requires full 3D modeling for adequate descripti This includes spatial and temporal resolution, but also the number of macro-particles per cell representing the target density for proper phase space sampling. You will need a computing cluster for adequate resolution of the target density, see comments in the input file. -.. warning:: - - It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. - The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. - Not setting these parameters can quickly lead to memory overflows. - Run --- @@ -30,7 +24,12 @@ This example can be run **either** as: * **Python** script: ``mpiexec -n 2 python3 PICMI_inputs_2d.py`` or * WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_2d`` -For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. +.. tip:: + + For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. + + The input option ``warpx_numprocs`` / ``warpx.numprocs`` needs to be adjusted for parallel :ref:`domain decomposition `, to match the number of MPI ranks used. + In order to use dynamic load balancing, use the :ref:`more general method ` of setting blocks. .. tab-set:: @@ -53,18 +52,18 @@ Analyze .. _fig-tnsa-ps-electrons-pinhole: .. figure:: https://user-images.githubusercontent.com/5416860/295003882-c755fd47-4bb3-4439-9319-c48214cbaafd.png - :alt: Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + :alt: Longitudinal phase space of forward-moving electrons in a 2 degree opening angle. :width: 100% - Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + Longitudinal phase space of forward-moving electrons in a 2 degree opening angle. .. _fig-tnsa-ps-protons-pinhole: .. figure:: https://user-images.githubusercontent.com/5416860/295003988-dea3dfb7-0d55-4616-b32d-061fb429f9ac.png - :alt: Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + :alt: Longitudinal phase space of forward-moving protons in a 2 degree opening angle. :width: 100% - Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + Longitudinal phase space of forward-moving protons in a 2 degree opening angle. Time-resolved phase electron space analysis as in :numref:`fig-tnsa-ps-electrons-pinhole` gives information about, e.g., how laser energy is locally converted into electron kinetic energy. Later in time, ion phase spaces like :numref:`fig-tnsa-ps-protons-pinhole` can reveal where accelerated ion populations originate. diff --git a/Examples/Physics_applications/laser_ion/plot_2d.py b/Examples/Physics_applications/laser_ion/plot_2d.py index e85782a7d23..736203e85ea 100644 --- a/Examples/Physics_applications/laser_ion/plot_2d.py +++ b/Examples/Physics_applications/laser_ion/plot_2d.py @@ -14,12 +14,12 @@ import os import re -from matplotlib.colors import TwoSlopeNorm import matplotlib.pyplot as plt import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import pandas as pd import scipy.constants as sc +from matplotlib.colors import TwoSlopeNorm +from openpmd_viewer import OpenPMDTimeSeries plt.rcParams.update({'font.size':16}) diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 08ebe23f828..d541c2393ca 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -12,9 +12,9 @@ # --- of electrons - leads to a decrease of the potential on the surface over the time # --- until reaching an equilibrium floating potential of ~144.5 V (*). -from mpi4py import MPI as mpi import numpy as np import scipy.constants as scc +from mpi4py import MPI as mpi from pywarpx import picmi from pywarpx.callbacks import installafterEsolve, installafterInitEsolve @@ -26,7 +26,8 @@ class SpaceChargeFieldCorrector(object): """ Class used by the callback functions to calculate the - correct charge on the spacecraft at each initialisation. + correct field around the spacecraft, at each timestep + (taking into account the charge that has been collected on the spacecraft) """ def __init__(self): self.saved_first_iteration_fields = False @@ -47,12 +48,12 @@ def correct_space_charge_fields(self, q=None): q = compute_actual_charge_on_spacecraft() # Correct fields so as to recover the actual charge - Er = ExWrapper(include_ghosts=True)[:,:] - Er[...] = Er[...]+(q - q_v)*self.normalized_Er[...] - Ez = EzWrapper(include_ghosts=True)[:,:] - Ez[...] += (q - q_v)*self.normalized_Ez[...] - phi = PhiFPWrapper(include_ghosts=True)[:,:] - phi[...] += (q - q_v)*self.normalized_phi[...] + Er = ExWrapper(include_ghosts=True) + Er[...] += (q - q_v)*self.normalized_Er + Ez = EzWrapper(include_ghosts=True) + Ez[...] += (q - q_v)*self.normalized_Ez + phi = PhiFPWrapper(include_ghosts=True) + phi[...] += (q - q_v)*self.normalized_phi self.spacecraft_potential += (q - q_v)*self.spacecraft_capacitance sim.extension.warpx.set_potential_on_eb( "%f" %self.spacecraft_potential ) print('Setting potential to %f' %self.spacecraft_potential) @@ -112,10 +113,7 @@ def compute_virtual_charge_on_spacecraft(): # Compute integral of rho over volume of the domain # (i.e. total charge of the plasma particles) - rho_integral = 0.0 - for k in range(1, nz-1): - for i in range(1, nr-1): - rho_integral += rho[i,k] * r[i] * dr * dz + rho_integral = (rho[1:nr-1,1:nz-1] * r[1:nr-1,np.newaxis]).sum()*dr*dz # Due to an oddity in WarpX (which will probably be solved later) # we need to multiply `rho` by `-epsilon_0` to get the correct charge diff --git a/Examples/Physics_applications/spacecraft_charging/analysis.py b/Examples/Physics_applications/spacecraft_charging/analysis.py index d8e8ac86af8..ef75fd1a10a 100755 --- a/Examples/Physics_applications/spacecraft_charging/analysis.py +++ b/Examples/Physics_applications/spacecraft_charging/analysis.py @@ -17,9 +17,9 @@ import matplotlib.pyplot as plt import numpy as np +import yt from openpmd_viewer import OpenPMDTimeSeries from scipy.optimize import curve_fit -import yt yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') @@ -61,8 +61,8 @@ def func(x, v0, tau): print('v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) -tolerance_v0=0.01 -tolerance_tau=0.01 +tolerance_v0=0.04 +tolerance_tau=0.04 print("tolerance for v0 = "+ str(tolerance_v0 *100) + '%') print("tolerance for tau = "+ str(tolerance_tau*100) + '%') diff --git a/Examples/Physics_applications/uniform_plasma/inputs_3d b/Examples/Physics_applications/uniform_plasma/inputs_3d index b0275f6eed6..83310088176 100644 --- a/Examples/Physics_applications/uniform_plasma/inputs_3d +++ b/Examples/Physics_applications/uniform_plasma/inputs_3d @@ -45,7 +45,7 @@ electrons.uz_th = 0.01 # uth the std of the (unitless) momentum diagnostics.diags_names = diag1 chk diag1.intervals = 4 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz w +diag1.electrons.variables = x y z ux uy uz w diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho chk.intervals = 6 diff --git a/Examples/Tests/AcceleratorLattice/analysis.py b/Examples/Tests/AcceleratorLattice/analysis.py index 55fb1b2ac96..5f3de4543d6 100755 --- a/Examples/Tests/AcceleratorLattice/analysis.py +++ b/Examples/Tests/AcceleratorLattice/analysis.py @@ -19,8 +19,8 @@ import sys import numpy as np -from scipy.constants import c, e, m_e import yt +from scipy.constants import c, e, m_e yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/AcceleratorLattice/inputs_quad_3d b/Examples/Tests/AcceleratorLattice/inputs_quad_3d index 2bc98f64118..d62b9b45a0e 100644 --- a/Examples/Tests/AcceleratorLattice/inputs_quad_3d +++ b/Examples/Tests/AcceleratorLattice/inputs_quad_3d @@ -2,8 +2,8 @@ max_step = 50 amr.n_cell = 8 8 32 amr.max_level = 0 geometry.dims = 3 -geometry.prob_lo = -0.2 -0.2 0. -geometry.prob_hi = +0.2 +0.2 1.6 +geometry.prob_lo = -0.2 -0.2 -0.025 +geometry.prob_hi = +0.2 +0.2 1.575 warpx.const_dt = 1e-9 warpx.do_electrostatic = labframe diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_2d.py index fa3299925a8..85faab61fcc 100755 --- a/Examples/Tests/Implicit/analysis_vandb_2d.py +++ b/Examples/Tests/Implicit/analysis_vandb_2d.py @@ -14,8 +14,8 @@ import sys import numpy as np -from scipy.constants import e, epsilon_0 import yt +from scipy.constants import e, epsilon_0 sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d index 50d28a2db75..465d9dd6965 100644 --- a/Examples/Tests/Implicit/inputs_1d +++ b/Examples/Tests/Implicit/inputs_1d @@ -71,8 +71,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit index 2271a0bb1bc..4008a559588 100644 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -71,8 +71,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_2d index 2dc57323efe..33ce964710a 100644 --- a/Examples/Tests/Implicit/inputs_vandb_2d +++ b/Examples/Tests/Implicit/inputs_vandb_2d @@ -82,8 +82,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 20 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index c0b03f4a20b..2b21184dc3d 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -21,8 +21,8 @@ import numpy as np import openpmd_api as io -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/boundaries/analysis.py b/Examples/Tests/boundaries/analysis.py index 0a879bb83e5..9c108b16196 100755 --- a/Examples/Tests/boundaries/analysis.py +++ b/Examples/Tests/boundaries/analysis.py @@ -18,8 +18,8 @@ import sys import numpy as np -from scipy.constants import c, m_e import yt +from scipy.constants import c, m_e yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/collision/analysis_collision_rz.py b/Examples/Tests/collision/analysis_collision_rz.py index 9f9b9c17f5e..b206b2eba7b 100755 --- a/Examples/Tests/collision/analysis_collision_rz.py +++ b/Examples/Tests/collision/analysis_collision_rz.py @@ -16,9 +16,9 @@ # tolerance: 1.0e-30 # Possible running time: ~ 1.0 s -from glob import glob import os import sys +from glob import glob import numpy as np import yt diff --git a/Examples/Tests/dive_cleaning/analysis.py b/Examples/Tests/dive_cleaning/analysis.py index 504368fa85e..a9b52455baa 100755 --- a/Examples/Tests/dive_cleaning/analysis.py +++ b/Examples/Tests/dive_cleaning/analysis.py @@ -22,8 +22,8 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc -from scipy.special import gammainc import yt +from scipy.special import gammainc yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index 097ea0674f7..e63a3f02ba8 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -18,11 +18,14 @@ inverse t(r) can be solved for exactly. """ import os +import re import sys import numpy as np -from scipy.optimize import fsolve import yt +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c +from scipy.optimize import fsolve sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI @@ -34,6 +37,15 @@ ds = yt.load( filename ) t_max = ds.current_time.item() # time of simulation +# Parse test name and check if particle_shape = 4 is used +emass_10 = True if re.search('emass_10', filename) else False + +if emass_10: + l2_tolerance = 0.096 + m_e = 10 +else: + l2_tolerance = 0.05 + m_e = 9.10938356e-31 #Electron mass in kg ndims = np.count_nonzero(ds.domain_dimensions > 1) if ndims == 2: @@ -58,7 +70,6 @@ # Constants eps_0 = 8.8541878128e-12 #Vacuum Permittivity in C/(V*m) -m_e = 9.10938356e-31 #Electron mass in kg q_e = -1.60217662e-19 #Electron charge in C pi = np.pi #Circular constant of the universe r_0 = 0.1 #Initial radius of sphere @@ -125,7 +136,6 @@ def calculate_error(E_axis, xmin, dx, nx): return L2_error - L2_error_x = calculate_error(Ex_axis, xmin, dx, nx) L2_error_y = calculate_error(Ey_axis, ymin, dy, ny) L2_error_z = calculate_error(Ez_axis, zmin, dz, nz) @@ -133,9 +143,24 @@ def calculate_error(E_axis, xmin, dx, nx): print("L2 error along y-axis = %s" %L2_error_y) print("L2 error along z-axis = %s" %L2_error_z) -assert L2_error_x < 0.05 -assert L2_error_y < 0.05 -assert L2_error_z < 0.05 +assert L2_error_x < l2_tolerance +assert L2_error_y < l2_tolerance +assert L2_error_z < l2_tolerance + +# Check conservation of energy +def return_energies(iteration): + ux, uy, uz, phi, m, q, w = ts.get_particle(['ux', 'uy', 'uz', 'phi', 'mass', 'charge', 'w'], iteration=iteration) + E_kinetic = (w*m*c**2 * (np.sqrt(1 + ux**2 + uy**2 + uz**2) - 1)).sum() + E_potential = 0.5*(w*q*phi).sum() # potential energy of particles in their own space-charge field: includes factor 1/2 + return E_kinetic, E_potential +ts = OpenPMDTimeSeries('./diags/diag2') +if 'phi' in ts.avail_record_components['electron']: + # phi is only available when this script is run with the labframe poisson solver + print('Checking conservation of energy') + Ek_i, Ep_i = return_energies(0) + Ek_f, Ep_f = return_energies(30) + assert Ep_f < 0.7*Ep_i # Check that potential energy changes significantly + assert abs( (Ek_i + Ep_i) - (Ek_f + Ep_f) ) < 0.003 * (Ek_i + Ep_i) # Check conservation of energy # Checksum regression analysis test_name = os.path.split(os.getcwd())[1] diff --git a/Examples/Tests/electrostatic_sphere/inputs_3d b/Examples/Tests/electrostatic_sphere/inputs_3d index afc6649f012..1f15b2da0de 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_3d +++ b/Examples/Tests/electrostatic_sphere/inputs_3d @@ -29,7 +29,13 @@ electron.profile = parse_density_function electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" electron.momentum_distribution_type = at_rest -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 diag2 + diag1.intervals = 30 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere/inputs_rz b/Examples/Tests/electrostatic_sphere/inputs_rz index 58f31046d89..2b6151e6d8c 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_rz +++ b/Examples/Tests/electrostatic_sphere/inputs_rz @@ -30,7 +30,14 @@ electron.profile = parse_density_function electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" electron.momentum_distribution_type = at_rest -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 diag2 + diag1.intervals = 30 diag1.diag_type = Full diag1.fields_to_plot = Er Et Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.electron.variables = x y z ux uy uz w phi +diag2.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis.py b/Examples/Tests/electrostatic_sphere_eb/analysis.py index c1a5ba33a1b..bf2725616b5 100755 --- a/Examples/Tests/electrostatic_sphere_eb/analysis.py +++ b/Examples/Tests/electrostatic_sphere_eb/analysis.py @@ -9,6 +9,7 @@ sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI + # Check reduced diagnostics for charge on EB import numpy as np from scipy.constants import epsilon_0 diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py b/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py index 5d807a2dbd3..18aba4322f0 100755 --- a/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py @@ -20,8 +20,8 @@ import sys import numpy as np -from unyt import m import yt +from unyt import m sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_cube/analysis_fields.py b/Examples/Tests/embedded_boundary_cube/analysis_fields.py index 13bd32d73e4..dc6af9d57d2 100755 --- a/Examples/Tests/embedded_boundary_cube/analysis_fields.py +++ b/Examples/Tests/embedded_boundary_cube/analysis_fields.py @@ -5,8 +5,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py b/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py index 67e893b199a..8faa299025e 100755 --- a/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py +++ b/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py @@ -4,8 +4,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_cube/inputs_2d b/Examples/Tests/embedded_boundary_cube/inputs_2d index 476992b360d..372e0dc0340 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_2d +++ b/Examples/Tests/embedded_boundary_cube/inputs_2d @@ -15,10 +15,14 @@ my_constants.xmin = -0.5 my_constants.zmin = -0.5 my_constants.xmax = 0.5 my_constants.zmax = 0.5 -# Alternatively one could use parser to build EB # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. warpx.eb_implicit_function = "max(max(x+xmin,-(x+xmax)), max(z+zmin,-(z+zmax)))" +# To test the initial electrostatic solver, we also add uniform potential +# Since, the potential is uniform here, it should not modify the fields, so this mainly +# tests that calling the electrostatic solver does not raise any error/crash +warpx.eb_potential(x,y,z,t) = "1" + warpx.B_ext_grid_init_style = parse_B_ext_grid_function warpx.E_ext_grid_init_style = parse_E_ext_grid_function diff --git a/Examples/Tests/embedded_boundary_cube/inputs_3d b/Examples/Tests/embedded_boundary_cube/inputs_3d index a98cd0b75ac..61eb1192e04 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_3d +++ b/Examples/Tests/embedded_boundary_cube/inputs_3d @@ -19,6 +19,11 @@ eb2.box_has_fluid_inside = true # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. # warpx.eb_implicit_function = "max(max(max(x-0.5,-0.5-x), max(y-0.5,-0.5-y)), max(z-0.5,-0.5-z))" +# To test the initial electrostatic solver, we also add uniform potential +# Since, the potential is uniform here, it should not modify the fields, so this mainly +# tests that calling the electrostatic solver does not raise any error/crash +warpx.eb_potential(x,y,z,t) = "1" + warpx.B_ext_grid_init_style = parse_B_ext_grid_function my_constants.m = 0 my_constants.n = 1 diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py index 1f7b604365c..e849958468f 100755 --- a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py @@ -11,8 +11,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py index 5c8c544d0c4..dcdbc83a729 100755 --- a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py @@ -4,8 +4,8 @@ import sys import numpy as np -from scipy.constants import c, mu_0, pi import yt +from scipy.constants import c, mu_0, pi sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py index 470cbfe6065..d0271f6aa94 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py @@ -26,9 +26,9 @@ import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, m_e, m_p from scipy.special import erf -import yt sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/initial_distribution/analysis_distribution.py b/Examples/Tests/initial_distribution/analysis_distribution.py index d5301c077d2..5a3774133db 100755 --- a/Examples/Tests/initial_distribution/analysis_distribution.py +++ b/Examples/Tests/initial_distribution/analysis_distribution.py @@ -22,9 +22,9 @@ import sys import numpy as np -from read_raw_data import read_reduced_diags, read_reduced_diags_histogram import scipy.constants as scc import scipy.special as scs +from read_raw_data import read_reduced_diags, read_reduced_diags_histogram sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/ion_stopping/analysis_ion_stopping.py b/Examples/Tests/ion_stopping/analysis_ion_stopping.py index b090014c0cf..d7774c14d6b 100755 --- a/Examples/Tests/ion_stopping/analysis_ion_stopping.py +++ b/Examples/Tests/ion_stopping/analysis_ion_stopping.py @@ -16,8 +16,8 @@ import sys import numpy as np -from scipy.constants import e, epsilon_0, k, m_e, m_p import yt +from scipy.constants import e, epsilon_0, k, m_e, m_p sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/langmuir/analysis_2d.py b/Examples/Tests/langmuir/analysis_2d.py index b3327703b82..94f97ca6de8 100755 --- a/Examples/Tests/langmuir/analysis_2d.py +++ b/Examples/Tests/langmuir/analysis_2d.py @@ -18,8 +18,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/langmuir/analysis_3d.py b/Examples/Tests/langmuir/analysis_3d.py index 2e56e64d641..68334f506ff 100755 --- a/Examples/Tests/langmuir/analysis_3d.py +++ b/Examples/Tests/langmuir/analysis_3d.py @@ -18,8 +18,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/langmuir/inputs_3d b/Examples/Tests/langmuir/inputs_3d index 432b4788171..604ed9bc514 100644 --- a/Examples/Tests/langmuir/inputs_3d +++ b/Examples/Tests/langmuir/inputs_3d @@ -93,5 +93,5 @@ diagnostics.diags_names = diag1 diag1.intervals = max_step diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho -diag1.electrons.variables = w ux -diag1.positrons.variables = uz +diag1.electrons.variables = x y z w ux +diag1.positrons.variables = x y z uz diff --git a/Examples/Tests/langmuir_fluids/analysis_2d.py b/Examples/Tests/langmuir_fluids/analysis_2d.py index cf5d2fb44de..f7244f87137 100755 --- a/Examples/Tests/langmuir_fluids/analysis_2d.py +++ b/Examples/Tests/langmuir_fluids/analysis_2d.py @@ -17,8 +17,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/langmuir_fluids/analysis_3d.py b/Examples/Tests/langmuir_fluids/analysis_3d.py index 0211956859e..686907f103a 100755 --- a/Examples/Tests/langmuir_fluids/analysis_3d.py +++ b/Examples/Tests/langmuir_fluids/analysis_3d.py @@ -18,8 +18,8 @@ import sys import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import yt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable yt.funcs.mylog.setLevel(50) diff --git a/Examples/Tests/laser_injection/analysis_2d.py b/Examples/Tests/laser_injection/analysis_2d.py index 37793171883..4424fe134bc 100755 --- a/Examples/Tests/laser_injection/analysis_2d.py +++ b/Examples/Tests/laser_injection/analysis_2d.py @@ -24,10 +24,10 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1 import make_axes_locatable import numpy as np -from scipy.signal import hilbert import yt +from mpl_toolkits.axes_grid1 import make_axes_locatable +from scipy.signal import hilbert sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py b/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py index a00d322bd18..44bbd00a7f4 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py +++ b/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py @@ -148,10 +148,7 @@ def launch_analysis(executable): def main() : from lasy.laser import Laser - from lasy.profiles import ( - CombinedLongitudinalTransverseProfile, - GaussianProfile, - ) + from lasy.profiles import CombinedLongitudinalTransverseProfile, GaussianProfile from lasy.profiles.longitudinal import GaussianLongitudinalProfile from lasy.profiles.transverse import LaguerreGaussianTransverseProfile diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index eec2ba4fffb..543eb62484a 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -40,8 +40,8 @@ ## the initial number of protons and borons. ## ## The third test corresponds to a Maxwellian plasma with a 44 keV temperature. The alpha yield is -## directly compared to the analytical fits of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 -## (2000) for a thermal plasma. +## directly compared to the analytical fits of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 +## (2023) for a thermal plasma. ## ## The fourth test corresponds to a plasma with an extremely small boron density, so that all boron ## macroparticles should have disappeared by the end of the simulation, which we verify. @@ -259,10 +259,10 @@ def check_isotropy(data, relative_tolerance): def astrophysical_factor_lowE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the low energy fit in the range E < 400 keV - ## described in equation (2) of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000) + ## described in equation (3) of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023) C0 = 197. - C1 = 0.24 - C2 = 2.31e-4 + C1 = 0.269 + C2 = 2.54e-4 AL = 1.82e4 EL = 148. dEL = 2.35 @@ -271,12 +271,12 @@ def astrophysical_factor_lowE(E): def astrophysical_factor_midE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the mid energy fit in the range - ## 400 keV < E < 642 keV described in equation (3) of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) - D0 = 330. - D1 = 66.1 - D2 = -20.3 - D5 = -1.58 + ## 400 keV < E < 668 keV described in equation (4) of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023) + D0 = 346. + D1 = 150. + D2 = -59.9 + D5 = -0.460 E_400 = 400. E_100 = 100. E_norm = (E - E_400)/E_100 @@ -285,29 +285,29 @@ def astrophysical_factor_midE(E): def astrophysical_factor_highE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the high energy fit in the range - ## 642 keV < E < 3500 keV described in equation (4) of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) - A0 = 2.57e6 - A1 = 5.67e5 - A2 = 1.34e5 - A3 = 5.68e5 - E0 = 581.3 - E1 = 1083. - E2 = 2405. - E3 = 3344. - dE0 = 85.7 - dE1 = 234. - dE2 = 138. - dE3 = 309. - B = 4.38 + ## 668 keV < E < 9760 keV described in equation (5) of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023) + A0 = 1.98e6 + A1 = 3.89e6 + A2 = 1.36e6 + A3 = 3.71e6 + E0 = 640.9 + E1 = 1211. + E2 = 2340. + E3 = 3294. + dE0 = 85.5 + dE1 = 414. + dE2 = 221. + dE3 = 351. + B = 0.381 return A0/((E-E0)**2 + dE0**2) + A1/((E-E1)**2 + dE1**2) + \ A2/((E-E2)**2 + dE2**2) + A3/((E-E3)**2 + dE3**2) + B def astrophysical_factor(E): ## E is in keV - ## Returns astrophysical factor in MeV b using the fits described in W.M. Nevins - ## and R. Swain, Nuclear Fusion, 40, 865 (2000) - conditions = [E <= 400, E <= 642, E > 642] + ## Returns astrophysical factor in MeV b using the fits described in A. Tentori + ## and F. Belloni, Nuclear Fusion, 63, 086001 (2023) + conditions = [E <= 400, E <= 668, E > 668] choices = [astrophysical_factor_lowE(E), astrophysical_factor_midE(E), astrophysical_factor_highE(E)] @@ -316,20 +316,20 @@ def astrophysical_factor(E): def pb_cross_section_buck_fit(E): ## E is in MeV ## Returns cross section in b using a power law fit of the data presented in Buck et al., - ## Nuclear Physics A, 398(2), 189-202 (1983) in the range E > 3.5 MeV. - E_start_fit = 3.5 + ## Nuclear Physics A, 398(2), 189-202 (1983) in the range E > 9.76 MeV. + E_start_fit = 9.76 ## Cross section at E = E_start_fit = 3.5 MeV - cross_section_start_fit = 0.2168440845211521 + cross_section_start_fit = 0.01277998 slope_fit = -2.661840717596765 return cross_section_start_fit*(E/E_start_fit)**slope_fit def pb_cross_section(E): ## E is in keV - ## Returns cross section in b using the fits described in W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) for E < 3.5 MeV and a power law fit of the data presented in - ## Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) for E > 3.5 MeV. + ## Returns cross section in b using the fits described in A. Tentori and F. Belloni, + ## Nucl. Fusion, 63, 086001 (2023) for E < 9.76 MeV otherwise returns a power law fit + ## of the data in Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) E_MeV = E/1.e3 - conditions = [E <= 3500, E > 3500] + conditions = [E <= 9760, E > 9760] choices = [astrophysical_factor(E)/E_MeV * np.exp(-np.sqrt(E_Gamow_MeV / E_MeV)), pb_cross_section_buck_fit(E_MeV)] return np.select(conditions, choices) @@ -598,13 +598,13 @@ def check_xy_isotropy(data): def sigmav_thermal_fit_lowE_nonresonant(T): ## Temperature T is in keV ## Returns the nonresonant average of cross section multiplied by relative velocity in m^3/s, - ## in the range T <= 70 keV, as described by equation 9 of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000). + ## in the range T <= 70 keV, as described by equations 10-14 of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023). E0 = (E_Gamow_keV/4.)**(1./3.) * T**(2./3.) DE0 = 4.*np.sqrt(T*E0/3.) C0 = 197.*1.e3 - C1 = 0.24*1.e3 - C2 = 2.31e-4*1.e3 + C1 = 0.269*1.e3 + C2 = 2.54e-4*1.e3 tau = 3.*E0/T Seff = C0*(1.+5./(12.*tau)) + C1*(E0+35./36.*T) + C2*(E0**2 + 89./36.*E0*T) ## nonresonant sigma times vrel, in barn meter per second @@ -615,21 +615,21 @@ def sigmav_thermal_fit_lowE_nonresonant(T): def sigmav_thermal_fit_lowE_resonant(T): ## Temperature T is in keV ## Returns the resonant average of cross section multiplied by relative velocity in m^3/s, - ## in the range T <= 70 keV, as described by equation 11 of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000). + ## in the range T <= 70 keV, as described by equation 15 of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023). return 5.41e-21 * np.exp(-148./T) / T**(3./2.) def sigmav_thermal_fit_lowE(T): ## Temperature T is in keV ## Returns the average of cross section multiplied by relative velocity in m^3/s, using the - ## fits described in section 3.1 of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). + ## fits described in section 2.2 of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fits are valid for T <= 70 keV. return sigmav_thermal_fit_lowE_nonresonant(T) + sigmav_thermal_fit_lowE_resonant(T) def expected_alpha_thermal(T, proton_density, boron_density, dV, dt): ## Computes the expected number of produced alpha particles when the protons and borons follow ## a Maxwellian distribution with a temperature T, in keV. This uses the thermal fits described - ## in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). + ## in A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fit used here is only valid in the range T <= 70 keV. assert((T >=0) and (T<=70)) diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d index 0943eee1d22..8099d92928b 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d @@ -11,6 +11,8 @@ geometry.dims = 3 geometry.prob_lo = 0. 0. 0. geometry.prob_hi = 8. 8. 16. +warpx.random_seed = 4 + ################################# ###### Boundary Condition ####### ################################# diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 4f3cd731323..ec67282fd88 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -12,8 +12,8 @@ import sys import dill -from mpi4py import MPI as mpi import numpy as np +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, picmi @@ -269,7 +269,7 @@ def setup_run(self): name='field_diag', grid=self.grid, period=self.total_steps, - data_list=['B', 'E'], + data_list=['B', 'E', 'J_displacement'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_EM_modes_1d_plt', # warpx_format = 'openpmd', diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 81ba65e5b28..89c5230a6ad 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -10,8 +10,8 @@ import sys import dill -from mpi4py import MPI as mpi import numpy as np +from mpi4py import MPI as mpi from pywarpx import picmi diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index 9004c24a05c..58e615c5332 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -3,11 +3,11 @@ # --- Analysis script for the hybrid-PIC example producing EM modes. import dill -from matplotlib import colors import matplotlib.pyplot as plt import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import scipy.fft as fft +from matplotlib import colors +from openpmd_viewer import OpenPMDTimeSeries from scipy.interpolate import RegularGridInterpolator from scipy.special import j1, jn, jn_zeros @@ -154,7 +154,7 @@ def process(it): amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) print("Amplitude sample: ", amps) assert np.allclose( - amps, np.array([61.50945919, 19.74831134, 101.01820349, 10.8974811]) + amps, np.array([ 61.02377286, 19.80026021, 100.47687017, 10.83331295]) ) if sim.test: diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index d545195a9e5..d4430502f42 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -11,8 +11,8 @@ import time import dill -from mpi4py import MPI as mpi import numpy as np +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index d9a71df6ac4..4f8b5edcc3e 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -12,8 +12,8 @@ import time import dill -from mpi4py import MPI as mpi import numpy as np +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index a7f2e0da12d..56173893b7f 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -8,13 +8,13 @@ # --- https://aip.scitation.org/doi/10.1063/1.4943893. import argparse -from pathlib import Path import shutil import sys +from pathlib import Path import dill -from mpi4py import MPI as mpi import numpy as np +from mpi4py import MPI as mpi from pywarpx import callbacks, fields, libwarpx, picmi diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py index e48d5c03fb2..0fb1c05ae1a 100755 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py @@ -5,9 +5,9 @@ import glob import dill -from matplotlib import colors import matplotlib.pyplot as plt import numpy as np +from matplotlib import colors plt.rcParams.update({'font.size': 20}) diff --git a/Examples/Tests/openbc_poisson_solver/analysis.py b/Examples/Tests/openbc_poisson_solver/analysis.py new file mode 100755 index 00000000000..dc7b5dca8bd --- /dev/null +++ b/Examples/Tests/openbc_poisson_solver/analysis.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import epsilon_0, pi +from scipy.special import erf + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +sigmaz = 300e-6 +sigmax = 516e-9 +sigmay = 7.7e-9 +Q = -3.2e-9 + +def w(z): + return np.exp(-z**2) * ( 1 + erf(1.j*z) ) + +def evaluate_E(x, y, z): + ''' Basseti-Erskine formula https://cds.cern.ch/record/122227/files/198005132.pdf ''' + den = np.sqrt(2*(sigmax**2-sigmay**2)) + arg1 = (x+1j*y)/den + term1 = w(arg1) + arg2 = (x*sigmay/sigmax + 1j*y*sigmax/sigmay)/den + term2 = -np.exp(-x**2/(2*sigmax**2)-y**2/(2*sigmay**2))*w(arg2) + factor = Q/(2.*np.sqrt(2.)*pi*epsilon_0*sigmaz*den)*np.exp(-z**2/(2*sigmaz**2)) + E_complex = factor*(term1 + term2) + return E_complex.imag, E_complex.real + + +fn = sys.argv[1] + +path=os.path.join('diags', 'diag2') +ts = OpenPMDTimeSeries(path) + +Ex, info = ts.get_field(field='E', coord='x', iteration=0, plot=False) +Ey, info = ts.get_field(field='E', coord='y', iteration=0, plot=False) + +grid_x = info.x[1:-1] +grid_y = info.y[1:-1] +grid_z = info.z[1:-1] + +hnx = int(0.5*len(grid_x)) +hny = int(0.5*len(grid_y)) + +# Compare theoretical and WarpX Ex, Ey fields for every z +for k, z in enumerate(grid_z,start=1): + Ex_warpx = Ex[k,hny,1:-1] + Ey_warpx = Ey[k,1:-1,hnx] + + Ex_theory = evaluate_E(grid_x, 0., z)[0] + Ey_theory = evaluate_E(0., grid_y, z)[1] + + assert(np.allclose(Ex_warpx, Ex_theory, rtol=0.032, atol=0)) + assert(np.allclose(Ey_warpx, Ey_theory, rtol=0.029, atol=0)) + + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-2) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/inputs_3d new file mode 100644 index 00000000000..2ac17a06751 --- /dev/null +++ b/Examples/Tests/openbc_poisson_solver/inputs_3d @@ -0,0 +1,45 @@ +my_constants.sigmaz = 300e-6 +my_constants.sigmax = 516e-9 +my_constants.sigmay = 7.7e-9 +my_constants.Q = 3.2e-9 + +max_step = 1 +amr.n_cell = 128 128 128 +amr.max_level = 0 +amr.max_grid_size = 128 +geometry.dims = 3 +geometry.prob_lo = -4*sigmax -4*sigmay -4*sigmaz +geometry.prob_hi = +4*sigmax +4*sigmay +4*sigmaz +boundary.field_lo = open open open +boundary.field_hi = open open open +warpx.const_dt = 1e-14 +warpx.do_electrostatic = relativistic +warpx.poisson_solver = fft +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 2 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "Q/(sqrt(2*pi)**3 * sigmax*sigmay*sigmaz * q_e) * exp( -x*x/(2*sigmax*sigmax) -y*y/(2*sigmay*sigmay) - z*z/(2*sigmaz*sigmaz) )" +electron.momentum_distribution_type = "constant" +electron.ux = 0.0 +electron.uy = 0.0 +electron.uz = 25000 +electron.initialize_self_fields = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho +diag1.format = plotfile + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Ex Ey +diag2.format = openpmd diff --git a/Examples/Tests/particle_boundary_interaction/analysis.py b/Examples/Tests/particle_boundary_interaction/analysis.py index 9768751ed74..ff83cc1fed7 100755 --- a/Examples/Tests/particle_boundary_interaction/analysis.py +++ b/Examples/Tests/particle_boundary_interaction/analysis.py @@ -11,8 +11,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py b/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py index ceb71ffb4b7..5e54fc42d87 100755 --- a/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py +++ b/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py @@ -16,8 +16,8 @@ import numpy as np import openpmd_api as io -from scipy.constants import c, e, m_e, m_p import yt +from scipy.constants import c, e, m_e, m_p sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/particle_thermal_boundary/analysis_2d.py b/Examples/Tests/particle_thermal_boundary/analysis_2d.py new file mode 100755 index 00000000000..db14479af2c --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/analysis_2d.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# +# Copyright 2023-2024 Revathi Jambunathan +# +# This file is part of WarpX +# +# License: BSD-3-Clause-LBNL + +""" +The script checks to see the growth in total field energy and total particle energy + +The input file in 2D initializes uniform plasma (electrons,ions) with +thermal boundary condition. We do not expect the particle energy to increase +beyond 2% in the time that it takes all particles to cross the domain boundary +""" + +import os +import sys + +import numpy as np + +sys.path.insert(1,'../../../../warpx/Regression/Checksum/') +import checksumAPI + +FE_rdiag = './diags/reducedfiles/EF.txt' +init_Fenergy = np.loadtxt(FE_rdiag)[1,2] +final_Fenergy = np.loadtxt(FE_rdiag)[-1,2] +assert(final_Fenergy/init_Fenergy < 40) +assert(final_Fenergy < 5.e-5) + +PE_rdiag = './diags/reducedfiles/EN.txt' +init_Penergy = np.loadtxt(PE_rdiag)[0,2] +final_Penergy = np.loadtxt(PE_rdiag)[-1,2] +assert( abs(final_Penergy - init_Penergy)/init_Penergy < 0.02) +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particle_thermal_boundary/inputs_2d b/Examples/Tests/particle_thermal_boundary/inputs_2d new file mode 100644 index 00000000000..9c3e18cd35d --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/inputs_2d @@ -0,0 +1,77 @@ +max_step = 2000 + +# number of grid points +amr.n_cell = 16 16 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 128 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = 0.e-6 0.e-6 # physical domain +geometry.prob_hi = 2.5e-7 2.5e-7 + +# Boundary condition +boundary.field_lo = pml pml +boundary.field_hi = pml pml +boundary.particle_lo = thermal thermal +boundary.particle_hi = thermal thermal +boundary.electrons.u_th = uth_e +boundary.C.u_th = uth_C +warpx.do_dive_cleaning = 0 + +# Verbosity +warpx.verbose = 1 + +# Order of particle shape factors +algo.particle_shape = 2 + +# CFL +warpx.cfl = 0.98 + +# Density +my_constants.n0 = 4e26 +my_constants.uth_e = 0.06256112470898544 +my_constants.uth_C = 0.00042148059678527106 + +# Particles +particles.species_names = electrons C + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = uth_e +electrons.uy_th = uth_e +electrons.uz_th = uth_e + +C.charge = 6*q_e +C.mass = 12*m_p +C.injection_style = "NUniformPerCell" +C.num_particles_per_cell_each_dim = 8 8 +C.profile = constant +C.density = n0/6 +C.momentum_distribution_type = gaussian +C.ux_th = uth_C +C.uy_th = uth_C +C.uz_th = uth_C + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 3000 +diag1.write_species = 0 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE +diag1.diag_type = Full + +warpx.reduced_diags_names = EN EF +EN.intervals = 10 +EN.type = ParticleEnergy +EF.intervals = 10 +EF.type = FieldEnergy diff --git a/Examples/Tests/photon_pusher/inputs_3d b/Examples/Tests/photon_pusher/inputs_3d index e2c75981e0e..3977a187e1b 100644 --- a/Examples/Tests/photon_pusher/inputs_3d +++ b/Examples/Tests/photon_pusher/inputs_3d @@ -140,19 +140,19 @@ p_dn_10.single_particle_weight = 1.0 diagnostics.diags_names = diag1 diag1.intervals = 50 diag1.diag_type = Full -diag1.p_xp_1.variables = ux uy uz -diag1.p_xn_1.variables = ux uy uz -diag1.p_yp_1.variables = ux uy uz -diag1.p_yn_1.variables = ux uy uz -diag1.p_zp_1.variables = ux uy uz -diag1.p_zn_1.variables = ux uy uz -diag1.p_dp_1.variables = ux uy uz -diag1.p_dn_1.variables = ux uy uz -diag1.p_xp_10.variables = ux uy uz -diag1.p_xn_10.variables = ux uy uz -diag1.p_yp_10.variables = ux uy uz -diag1.p_yn_10.variables = ux uy uz -diag1.p_zp_10.variables = ux uy uz -diag1.p_zn_10.variables = ux uy uz -diag1.p_dp_10.variables = ux uy uz -diag1.p_dn_10.variables = ux uy uz +diag1.p_xp_1.variables = x y z ux uy uz +diag1.p_xn_1.variables = x y z ux uy uz +diag1.p_yp_1.variables = x y z ux uy uz +diag1.p_yn_1.variables = x y z ux uy uz +diag1.p_zp_1.variables = x y z ux uy uz +diag1.p_zn_1.variables = x y z ux uy uz +diag1.p_dp_1.variables = x y z ux uy uz +diag1.p_dn_1.variables = x y z ux uy uz +diag1.p_xp_10.variables = x y z ux uy uz +diag1.p_xn_10.variables = x y z ux uy uz +diag1.p_yp_10.variables = x y z ux uy uz +diag1.p_yn_10.variables = x y z ux uy uz +diag1.p_zp_10.variables = x y z ux uy uz +diag1.p_zn_10.variables = x y z ux uy uz +diag1.p_dp_10.variables = x y z ux uy uz +diag1.p_dn_10.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/analysis.py b/Examples/Tests/plasma_lens/analysis.py index 9ce82d903f8..212e71087f9 100755 --- a/Examples/Tests/plasma_lens/analysis.py +++ b/Examples/Tests/plasma_lens/analysis.py @@ -19,8 +19,8 @@ import sys import numpy as np -from scipy.constants import c, e, m_e import yt +from scipy.constants import c, e, m_e yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/plasma_lens/inputs_3d b/Examples/Tests/plasma_lens/inputs_3d index 7763013adb1..a87c1eb7f44 100644 --- a/Examples/Tests/plasma_lens/inputs_3d +++ b/Examples/Tests/plasma_lens/inputs_3d @@ -48,4 +48,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_boosted_3d b/Examples/Tests/plasma_lens/inputs_boosted_3d index ecdf3b3baec..fa18ac439c4 100644 --- a/Examples/Tests/plasma_lens/inputs_boosted_3d +++ b/Examples/Tests/plasma_lens/inputs_boosted_3d @@ -50,4 +50,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_lattice_3d b/Examples/Tests/plasma_lens/inputs_lattice_3d index a41ca8baee9..94e57de8290 100644 --- a/Examples/Tests/plasma_lens/inputs_lattice_3d +++ b/Examples/Tests/plasma_lens/inputs_lattice_3d @@ -70,4 +70,4 @@ plasmalens4.dEdx = 200000. diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_short_3d b/Examples/Tests/plasma_lens/inputs_short_3d index 8fa111df3c9..5c91ab26498 100644 --- a/Examples/Tests/plasma_lens/inputs_short_3d +++ b/Examples/Tests/plasma_lens/inputs_short_3d @@ -52,4 +52,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 1 #84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py index cb1fc23ee62..042fc811a62 100755 --- a/Examples/Tests/point_of_contact_EB/analysis.py +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -11,8 +11,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/point_of_contact_EB/inputs_3d b/Examples/Tests/point_of_contact_EB/inputs_3d index 005733487e5..8b190c056ff 100644 --- a/Examples/Tests/point_of_contact_EB/inputs_3d +++ b/Examples/Tests/point_of_contact_EB/inputs_3d @@ -12,12 +12,6 @@ geometry.prob_hi = 0.26 0.26 0.26 boundary.field_lo = pec pec pec boundary.field_hi = pec pec pec -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 warpx.const_dt = 1e-10 warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.2**2)" diff --git a/Examples/Tests/point_of_contact_EB/inputs_rz b/Examples/Tests/point_of_contact_EB/inputs_rz index a492c1f705a..169a58adb94 100644 --- a/Examples/Tests/point_of_contact_EB/inputs_rz +++ b/Examples/Tests/point_of_contact_EB/inputs_rz @@ -12,12 +12,6 @@ geometry.prob_hi = 0.26 0.26 boundary.field_lo = none periodic boundary.field_hi = pec periodic -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 warpx.const_dt = 1e-10 warpx.eb_implicit_function = "-(x**2 -0.2**2)" diff --git a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py index 7104963c752..db1cc7dcad8 100755 --- a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py +++ b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import matplotlib.pyplot as plt -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable import numpy as np +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable from pywarpx import picmi diff --git a/Examples/Tests/qed/breit_wheeler/inputs_2d b/Examples/Tests/qed/breit_wheeler/inputs_2d index 7f507dd08d7..857b3243ac6 100644 --- a/Examples/Tests/qed/breit_wheeler/inputs_2d +++ b/Examples/Tests/qed/breit_wheeler/inputs_2d @@ -201,19 +201,19 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthBW -diag1.p2.variables = ux uy uz w opticalDepthBW -diag1.p3.variables = ux uy uz w opticalDepthBW -diag1.p4.variables = ux uy uz w opticalDepthBW - -diag1.ele1.variables = ux uy uz w opticalDepthQSR -diag1.ele2.variables = ux uy uz w opticalDepthQSR -diag1.ele3.variables = ux uy uz w opticalDepthQSR -diag1.ele4.variables = ux uy uz w opticalDepthQSR - -diag1.pos1.variables = ux uy uz w opticalDepthQSR -diag1.pos2.variables = ux uy uz w opticalDepthQSR -diag1.pos3.variables = ux uy uz w opticalDepthQSR -diag1.pos4.variables = ux uy uz w opticalDepthQSR +diag1.p1.variables = x z ux uy uz w opticalDepthBW +diag1.p2.variables = x z ux uy uz w opticalDepthBW +diag1.p3.variables = x z ux uy uz w opticalDepthBW +diag1.p4.variables = x z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x z ux uy uz w opticalDepthQSR diag1.format = plotfile diff --git a/Examples/Tests/qed/breit_wheeler/inputs_3d b/Examples/Tests/qed/breit_wheeler/inputs_3d index 2749a60b888..b79dfd9c030 100644 --- a/Examples/Tests/qed/breit_wheeler/inputs_3d +++ b/Examples/Tests/qed/breit_wheeler/inputs_3d @@ -201,19 +201,19 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthBW -diag1.p2.variables = ux uy uz w opticalDepthBW -diag1.p3.variables = ux uy uz w opticalDepthBW -diag1.p4.variables = ux uy uz w opticalDepthBW - -diag1.ele1.variables = ux uy uz w opticalDepthQSR -diag1.ele2.variables = ux uy uz w opticalDepthQSR -diag1.ele3.variables = ux uy uz w opticalDepthQSR -diag1.ele4.variables = ux uy uz w opticalDepthQSR - -diag1.pos1.variables = ux uy uz w opticalDepthQSR -diag1.pos2.variables = ux uy uz w opticalDepthQSR -diag1.pos3.variables = ux uy uz w opticalDepthQSR -diag1.pos4.variables = ux uy uz w opticalDepthQSR +diag1.p1.variables = x y z ux uy uz w opticalDepthBW +diag1.p2.variables = x y z ux uy uz w opticalDepthBW +diag1.p3.variables = x y z ux uy uz w opticalDepthBW +diag1.p4.variables = x y z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x y z ux uy uz w opticalDepthQSR diag1.format = plotfile diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_2d b/Examples/Tests/qed/quantum_synchrotron/inputs_2d index 2fc00231280..2ac2c782ccd 100644 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_2d +++ b/Examples/Tests/qed/quantum_synchrotron/inputs_2d @@ -178,12 +178,12 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthQSR -diag1.p2.variables = ux uy uz w opticalDepthQSR -diag1.p3.variables = ux uy uz w opticalDepthQSR -diag1.p4.variables = ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = ux uy uz w opticalDepthBW -diag1.qsp_2.variables = ux uy uz w opticalDepthBW -diag1.qsp_3.variables = ux uy uz w opticalDepthBW -diag1.qsp_4.variables = ux uy uz w opticalDepthBW +diag1.p1.variables = x z ux uy uz w opticalDepthQSR +diag1.p2.variables = x z ux uy uz w opticalDepthQSR +diag1.p3.variables = x z ux uy uz w opticalDepthQSR +diag1.p4.variables = x z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_3d b/Examples/Tests/qed/quantum_synchrotron/inputs_3d index 9bc963f7e94..429666ef938 100644 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_3d +++ b/Examples/Tests/qed/quantum_synchrotron/inputs_3d @@ -178,12 +178,12 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthQSR -diag1.p2.variables = ux uy uz w opticalDepthQSR -diag1.p3.variables = ux uy uz w opticalDepthQSR -diag1.p4.variables = ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = ux uy uz w opticalDepthBW -diag1.qsp_2.variables = ux uy uz w opticalDepthBW -diag1.qsp_3.variables = ux uy uz w opticalDepthBW -diag1.qsp_4.variables = ux uy uz w opticalDepthBW +diag1.p1.variables = x y z ux uy uz w opticalDepthQSR +diag1.p2.variables = x y z ux uy uz w opticalDepthQSR +diag1.p3.variables = x y z ux uy uz w opticalDepthQSR +diag1.p4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x y z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py b/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py index 93352d29ff3..21359ed8171 100755 --- a/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py +++ b/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py @@ -15,11 +15,11 @@ import sys import numpy as np +import yt from scipy.constants import c from scipy.constants import epsilon_0 as eps0 from scipy.constants import m_e, m_p from scipy.constants import mu_0 as mu0 -import yt sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d b/Examples/Tests/relativistic_space_charge_initialization/inputs_3d index 91cd6000547..c472f88c007 100644 --- a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d +++ b/Examples/Tests/relativistic_space_charge_initialization/inputs_3d @@ -1,5 +1,5 @@ max_step = 1 -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_grid_size = 32 amr.max_level = 0 diff --git a/Examples/Tests/repelling_particles/analysis_repelling.py b/Examples/Tests/repelling_particles/analysis_repelling.py index 0aa3ed53d1b..ebff010fc40 100755 --- a/Examples/Tests/repelling_particles/analysis_repelling.py +++ b/Examples/Tests/repelling_particles/analysis_repelling.py @@ -29,8 +29,8 @@ import matplotlib.pyplot as plt import numpy as np -from scipy.constants import c, m_e, physical_constants import yt +from scipy.constants import c, m_e, physical_constants yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/resampling/analysis_leveling_thinning.py b/Examples/Tests/resampling/analysis_leveling_thinning.py index 50fec628200..5f3dc8ecdff 100755 --- a/Examples/Tests/resampling/analysis_leveling_thinning.py +++ b/Examples/Tests/resampling/analysis_leveling_thinning.py @@ -13,8 +13,8 @@ import sys import numpy as np -from scipy.special import erf import yt +from scipy.special import erf sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning new file mode 100644 index 00000000000..1610c8b1221 --- /dev/null +++ b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning @@ -0,0 +1,46 @@ +max_step = 4 +warpx.verbose = 1 +warpx.const_dt = 1e-10 +amr.n_cell = 256 +amr.max_grid_size = 64 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = 0 +geometry.prob_hi = 0.1 + +# Boundary condition and field solver +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic +algo.particle_shape = 1 +algo.maxwell_solver = none + +particles.species_names = hydrogen +hydrogen.mass = 1.67262192369e-27 +hydrogen.charge = 1.602176634e-19 +hydrogen.injection_style = nrandompercell +hydrogen.initialize_self_fields = 0 +hydrogen.do_not_push = 1 +hydrogen.do_resampling = 1 +hydrogen.resampling_min_ppc = 10 +hydrogen.resampling_trigger_intervals = 1::2 +hydrogen.resampling_algorithm = velocity_coincidence_thinning +hydrogen.resampling_algorithm_delta_ur = 100000000.0 +hydrogen.resampling_algorithm_n_phi = 3 +hydrogen.resampling_algorithm_n_theta = 120 +hydrogen.num_particles_per_cell = 2500 +hydrogen.momentum_distribution_type = gaussian +hydrogen.ux_m = 0.0 +hydrogen.uy_m = 0.0 +hydrogen.uz_m = 0.0 +hydrogen.ux_th = 0.000326 +hydrogen.uy_th = 0.000326 +hydrogen.uz_th = 0.000326 +hydrogen.profile = constant +hydrogen.density = 1e+19 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 4 +diag1.diag_type = Full diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian new file mode 100644 index 00000000000..97830d8bb9e --- /dev/null +++ b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian @@ -0,0 +1,45 @@ +max_step = 4 +warpx.verbose = 1 +warpx.const_dt = 1e-10 +amr.n_cell = 256 +amr.max_grid_size = 64 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = 0 +geometry.prob_hi = 0.1 + +# Boundary condition and field solver +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic +algo.particle_shape = 1 +algo.maxwell_solver = none + +particles.species_names = hydrogen +hydrogen.mass = 1.67262192369e-27 +hydrogen.charge = 1.602176634e-19 +hydrogen.injection_style = nrandompercell +hydrogen.initialize_self_fields = 0 +hydrogen.do_not_push = 1 +hydrogen.do_resampling = 1 +hydrogen.resampling_min_ppc = 10 +hydrogen.resampling_trigger_intervals = 1::2 +hydrogen.resampling_algorithm = velocity_coincidence_thinning +hydrogen.resampling_algorithm_velocity_grid_type = cartesian +hydrogen.resampling_algorithm_delta_u = 10000000.0 5000000.0 7000000.0 +hydrogen.num_particles_per_cell = 2500 +hydrogen.momentum_distribution_type = gaussian +hydrogen.ux_m = 0.0 +hydrogen.uy_m = 0.0 +hydrogen.uz_m = 0.0 +hydrogen.ux_th = 0.000326 +hydrogen.uy_th = 0.000326 +hydrogen.uz_th = 0.000326 +hydrogen.profile = constant +hydrogen.density = 1e+19 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 4 +diag1.diag_type = Full diff --git a/Examples/Tests/resampling/inputs_leveling_thinning b/Examples/Tests/resampling/inputs_leveling_thinning index 9ed49ae358e..90e0d6802d6 100644 --- a/Examples/Tests/resampling/inputs_leveling_thinning +++ b/Examples/Tests/resampling/inputs_leveling_thinning @@ -57,7 +57,7 @@ resampled_part2.do_resampling = 1 resampled_part2.resampling_algorithm = leveling_thinning resampled_part2.resampling_algorithm_target_ratio = 1.3 # This should prevent actual resampling at timestep 7 -resampled_part2.resampling_algorithm_min_ppc = 80000 +resampled_part2.resampling_min_ppc = 80000 # This should trigger resampling at timestep 6 and 7 in this case. The rest is here to test the # intervals parser syntax. resampled_part2.resampling_trigger_intervals = 100 :120:3, 6:6:6 , 7 diff --git a/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py b/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py index ccb55183241..fd51298816b 100755 --- a/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py +++ b/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py @@ -25,7 +25,6 @@ import numpy as np import openpmd_api as io -from scipy.constants import m_e import yt yt.funcs.mylog.setLevel(0) diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 7d40bd5edf4..193b2575992 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -24,8 +24,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries sys.path.insert(1, '../../../../warpx/Regression/Checksum/') import checksumAPI diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py index dd13b993dc5..a4e0dafddbc 100755 --- a/Examples/Tests/scraping/analysis_rz_filter.py +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -23,8 +23,8 @@ import sys import numpy as np -from openpmd_viewer import OpenPMDTimeSeries import yt +from openpmd_viewer import OpenPMDTimeSeries tolerance = 0 diff --git a/Examples/Tests/space_charge_initialization/analysis.py b/Examples/Tests/space_charge_initialization/analysis.py index 0df21e6d984..48e19ea0b75 100755 --- a/Examples/Tests/space_charge_initialization/analysis.py +++ b/Examples/Tests/space_charge_initialization/analysis.py @@ -20,8 +20,8 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc -from scipy.special import gammainc import yt +from scipy.special import gammainc yt.funcs.mylog.setLevel(0) sys.path.insert(1, '../../../../warpx/Regression/Checksum/') diff --git a/Examples/Tests/space_charge_initialization/inputs_3d b/Examples/Tests/space_charge_initialization/inputs_3d index a24b67ff02e..c8058fac519 100644 --- a/Examples/Tests/space_charge_initialization/inputs_3d +++ b/Examples/Tests/space_charge_initialization/inputs_3d @@ -1,5 +1,5 @@ max_step = 1 -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_grid_size = 32 amr.max_level = 0 diff --git a/Examples/Tests/vay_deposition/analysis.py b/Examples/Tests/vay_deposition/analysis.py index f1680742ded..cfd089f2112 100755 --- a/Examples/Tests/vay_deposition/analysis.py +++ b/Examples/Tests/vay_deposition/analysis.py @@ -10,8 +10,8 @@ import sys import numpy as np -from scipy.constants import epsilon_0 import yt +from scipy.constants import epsilon_0 yt.funcs.mylog.setLevel(50) diff --git a/GNUmakefile b/GNUmakefile index d11d1f85cce..8c657025740 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -30,7 +30,6 @@ USE_GPU = FALSE EBASE = main -USE_GPUCLOCK = TRUE USE_PYTHON_MAIN = FALSE USE_SENSEI_INSITU = FALSE diff --git a/GOVERNANCE.rst b/GOVERNANCE.rst new file mode 100644 index 00000000000..b5253b80f9f --- /dev/null +++ b/GOVERNANCE.rst @@ -0,0 +1,144 @@ +.. _governance: + +WarpX Governance +================ + +WarpX is led in an open governance model, described in this file. + + +Steering Committee +------------------ + +Current Roster +^^^^^^^^^^^^^^ + +- Jean-Luc Vay (chair) +- Remi Lehe +- Axel Huebl + +See: `GitHub team `__ + +Role +^^^^ + +Members of the steering committee (SC) can change organizational settings, do administrative operations such as rename/move/archive repositories, change branch protection rules, etc. +SC members can call votes for decisions (technical or governance). + +The SC can veto decisions of the technical committee (TC) by voting in the SC. +The TC can overwrite a veto with a 2/3rd majority vote in the TC. +Decisions are documented in the `weekly developer meeting notes `__ and/or on the GitHub repository. + +The SC can change the governance structure, but only in a unanimous vote. + +Decision Process +^^^^^^^^^^^^^^^^ + +Decision of the SC usually happen in the weekly developer meetings, via e-mail or public chat. + +Decisions are made in a non-confidential manner, by majority on the cast votes of SC members. +Votes can be cast in asynchronous manner, e.g., over the time of 1-2 weeks. +In tie situations, the chair of the SC acts as the tie breaker. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +Appointed by current SC members in an unanimous vote. +As a SC member, regularly attending and contributing to the weekly developer meetings is expected. + +SC members can resign or be removed by majority vote, e.g., due to inactivity, bad acting or other reasons. + + +Technical Committee +------------------- + +Current Roster +^^^^^^^^^^^^^^ + +- Luca Fedeli +- Roelof Groenewald +- David Grote +- Axel Huebl +- Revathi Jambunathan +- Remi Lehe +- Andrew Myers +- Maxence Thévenet +- Jean-Luc Vay +- Weiqun Zhang +- Edoardo Zoni + +See: `GitHub team `__ + +Role +^^^^ + +The technical committee (TC) is the core governance body, where under normal operations most ideas are discussed and decisions are made. +Individual TC members can approve and merge code changes. +Usually, they seek approval by another maintainer for their own changes, too. +TC members lead - and weigh in on - technical discussions and, if needed, can call for a vote between TC members for a technical decision. +TC members merge/close PRs and issues, and moderate (including block/mute) bad actors. +The TC can propose governance changes to the SC. + + +Decision Process +^^^^^^^^^^^^^^^^ + +Discussion in the TC usually happens in the weekly developer meetings. + +If someone calls for a vote to make a decision: majority based on the cast votes; we need 50% of the committee participating to vote. In the absence of a quorum, the SC will decide according to its voting rules. + +Votes are cast in a non-confidential manner. +Decisions are documented in the `weekly developer meeting notes `__ and/or on the GitHub repository. + +TC members can individually appoint new contributors, unless a vote is called on an individual. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +TC members are the maintainers of WarpX. +As a TC member, regularly attending and contributing to the weekly developer meetings is expected. + +One is appointed to the TC by the steering committee, in a unanimous vote, or by majority vote of the TC. The SC can veto appointments. +Steering committee members can also be TC members. + +TC members can resign or be removed by majority vote by either TC or SC, e.g., due to inactivity, bad acting or other reasons. + + +Contributors +------------ + +Current Roster +^^^^^^^^^^^^^^ + +See: `GitHub team `__ + +Role +^^^^ + +Contributors are valuable, vetted developers of WarpX. +Contributions can be in many forms and not all need to be code contributions. +Examples include code pull requests, support in issues & user discussions, writing and updating documentation, writing tutorials, visualizations, R&D on algorithms, testing and benchmarking, etc. +Contributors can participate in developer meetings and weigh in on discussions. +Contributors can "triage" (add labels) to pull requests, issues, and GitHub discussion pages. +Contributors can comment and review PRs (but not merge). + +Decision Process +^^^^^^^^^^^^^^^^ + +Contributors can individually decide on classification (triage) of pull requests, issues, and GitHub discussion pages. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +Appointed after contributing to WarpX (see above) by any member of the TC. + +The role can be lost by resigning or by decision of an individual TC or SC member, e.g., due to inactivity, bad acting or other. + + +Former Members +-------------- + +"Former members" are the giants on whose shoulders we stand. +But, for the purpose of WarpX governance, they are *not* tracked as a governance role in WarpX. +Instead, former (e.g., inactive) contributors are acknowledged separately in GitHub contributor tracking, the WarpX documentation, references, citable Zenodo archives of releases, etc. as appropriate. + +Former members of SC, TC and Contributors are not kept in the roster, since committee role rosters shall reflect currently active members and the responsible governance body. diff --git a/LICENSE.txt b/LICENSE.txt index 9d9b939f342..2965985ebb1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,16 +1,44 @@ -WarpX Copyright (c) 2018, The Regents of the University of California, through Lawrence Berkeley National Laboratory, and Lawrence Livermore National Security, LLC, for the operation of Lawrence Livermore National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. - - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - -(1) Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -(2) Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -(3) Neither the name of the University of California, Lawrence Berkeley National Laboratory, Lawrence Livermore National Security, LLC, Lawrence Livermore National Laboratory, U.S. Dept. of Energy, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -You are under no obligation whatsoever to provide any bug fixes, patches, or upgrades to the features, functionality or performance of the source code ("Enhancements") to anyone; however, if you choose to make your Enhancements available either publicly, or directly to Lawrence Berkeley National Laboratory, without imposing a separate written license agreement for such Enhancements, then you hereby grant the following license: a non-exclusive, royalty-free perpetual license to install, use, modify, prepare derivative works, incorporate into other computer software, distribute, and sublicense such enhancements or derivative works thereof, in binary and source code form. +WarpX Copyright (c) 2018, The Regents of the University of California, +through Lawrence Berkeley National Laboratory, and Lawrence Livermore National +Security, LLC, for the operation of Lawrence Livermore National Laboratory +(subject to receipt of any required approvals from the U.S. Dept. of Energy). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +(1) Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +(2) Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +(3) Neither the name of the University of California, Lawrence Berkeley +National Laboratory, U.S. Dept. of Energy nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +You are under no obligation whatsoever to provide any bug fixes, patches, +or upgrades to the features, functionality or performance of the source +code ("Enhancements") to anyone; however, if you choose to make your +Enhancements available either publicly, or directly to Lawrence Berkeley +National Laboratory, without imposing a separate written license agreement +for such Enhancements, then you hereby grant the following license: a +non-exclusive, royalty-free perpetual license to install, use, modify, +prepare derivative works, incorporate into other computer software, +distribute, and sublicense such enhancements or derivative works thereof, +in binary and source code form. diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000000..e556dddec6c --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,19 @@ +This Software was developed under funding from the U.S. Department +of Energy and the U.S. Government consequently retains certain rights. As +such, the U.S. Government has been granted for itself and others acting on +its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the +Software to reproduce, distribute copies to the public, prepare derivative +works, and perform publicly and display publicly, and to permit other to do +so. + +Reference herein to any specific commercial product, process, or service +by trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the +United States Government, The Regents of the University of California, or +Lawrence Berkeley National Laboratory. + +The views and opinions of authors expressed herein do not necessarily +state or reflect those of the United States Government, +The Regents of the University of California, or +Lawrence Berkeley National Laboratory, and shall not be used for advertising +or product endorsement purposes. diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 3fa59285f26..6b24b35e918 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -9,6 +9,7 @@ import sys from . import Particles +from ._libwarpx import libwarpx from .Algo import algo from .Amr import amr from .Amrex import amrex @@ -22,9 +23,8 @@ from .HybridPICModel import hybridpicmodel from .Interpolation import interpolation from .Lasers import lasers, lasers_list -from .PSATD import psatd from .Particles import particles, particles_list -from ._libwarpx import libwarpx +from .PSATD import psatd class WarpX(Bucket): diff --git a/Python/pywarpx/__init__.py b/Python/pywarpx/__init__.py index d5181a0307c..037598f4ed4 100644 --- a/Python/pywarpx/__init__.py +++ b/Python/pywarpx/__init__.py @@ -23,6 +23,7 @@ if os.path.exists(p_abs): os.add_dll_directory(p_abs) +from ._libwarpx import libwarpx from .Algo import algo from .Amr import amr from .Amrex import amrex @@ -36,10 +37,9 @@ from .Interpolation import interpolation from .Lasers import lasers from .LoadThirdParty import load_cupy -from .PSATD import psatd from .Particles import newspecies, particles +from .PSATD import psatd from .WarpX import warpx -from ._libwarpx import libwarpx # This is a circular import and must happen after the import of libwarpx from . import picmi # isort:skip diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index 8f3c1dc5e2c..1f32fee6620 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -30,6 +30,8 @@ * ``beforeEsolve``: before the solve for E fields * ``poissonsolver``: In place of the computePhi call but only in an electrostatic simulation * ``afterEsolve``: after the solve for E fields +* ``afterBpush``: after the B field advance for electromagnetic solvers +* ``afterEpush``: after the E field advance for electromagnetic solvers * ``beforedeposition``: before the particle deposition (for charge and/or current) * ``afterdeposition``: after particle deposition (for charge and/or current) * ``beforestep``: before the time step @@ -276,6 +278,8 @@ def callfuncsinlist(self,*args,**kw): "beforeEsolve": {}, "poissonsolver": {'singlefunconly': True}, # external Poisson solver "afterEsolve": {}, + "afterBpush": {}, + "afterEpush": {}, "beforedeposition": {}, "afterdeposition": {}, "particlescraper": {}, @@ -401,6 +405,20 @@ def callfromafterEsolve(f): def installafterEsolve(f): installcallback('afterEsolve', f) +# ---------------------------------------------------------------------------- +def callfromafterBpush(f): + installcallback('afterBpush', f) + return f +def installafterBpush(f): + installcallback('afterBpush', f) + +# ---------------------------------------------------------------------------- +def callfromafterEpush(f): + installcallback('afterEpush', f) + return f +def installafterEpush(f): + installcallback('afterEpush', f) + # ---------------------------------------------------------------------------- def callfrombeforedeposition(f): installcallback('beforedeposition', f) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index f8e687f99bd..70471e24d1a 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -8,8 +8,8 @@ import numpy as np -from .LoadThirdParty import load_cupy from ._libwarpx import libwarpx +from .LoadThirdParty import load_cupy class ParticleContainerWrapper(object): @@ -42,17 +42,18 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, The type of species for which particles will be added x, y, z : arrays or scalars - The particle positions (default = 0.) + The particle positions (m) (default = 0.) ux, uy, uz : arrays or scalars - The particle momenta (default = 0.) + The particle proper velocities (m/s) (default = 0.) w : array or scalars Particle weights (default = 0.) unique_particles : bool - Whether the particles are unique or duplicated on several processes - (default = True) + True means the added particles are duplicated by each process; + False means the number of added particles is independent of + the number of processes (default = True) kwargs : dict Containing an entry for all the extra particle attribute arrays. If diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 959db6c701d..e7575f83b26 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -8,9 +8,9 @@ """Classes following the PICMI standard """ -from dataclasses import dataclass import os import re +from dataclasses import dataclass import numpy as np import periodictable @@ -133,12 +133,41 @@ class Species(picmistandard.PICMI_Species): warpx_do_resampling: bool, default=False Whether particles will be resampled + warpx_resampling_min_ppc: int, default=1 + Cells with fewer particles than this number will be + skipped during resampling. + warpx_resampling_trigger_intervals: bool, default=0 Timesteps at which to resample warpx_resampling_trigger_max_avg_ppc: int, default=infinity Resampling will be done when the average number of particles per cell exceeds this number + + warpx_resampling_algorithm: str, default="leveling_thinning" + Resampling algorithm to use. + + warpx_resampling_algorithm_velocity_grid_type: str, default="spherical" + Type of grid to use when clustering particles in velocity space. Only + applicable with the `velocity_coincidence_thinning` algorithm. + + warpx_resampling_algorithm_delta_ur: float + Size of velocity window used for clustering particles during grid-based + merging, with `velocity_grid_type == "spherical"`. + + warpx_resampling_algorithm_n_theta: int + Number of bins to use in theta when clustering particle velocities + during grid-based merging, with `velocity_grid_type == "spherical"`. + + warpx_resampling_algorithm_n_phi: int + Number of bins to use in phi when clustering particle velocities + during grid-based merging, with `velocity_grid_type == "spherical"`. + + warpx_resampling_algorithm_delta_u: array of floats or float + Size of velocity window used in ux, uy and uz for clustering particles + during grid-based merging, with `velocity_grid_type == "cartesian"`. If + a single number is given the same du value will be used in all three + directions. """ def init(self, kw): @@ -216,8 +245,17 @@ def init(self, kw): # Resampling settings self.do_resampling = kw.pop('warpx_do_resampling', None) + self.resampling_algorithm = kw.pop('warpx_resampling_algorithm', None) + self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) + self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) + self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) + self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) + self.resampling_algorithm_delta_u = kw.pop('warpx_resampling_algorithm_delta_u', None) + if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: + self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 def species_initialize_inputs(self, layout, initialize_self_fields = False, @@ -256,8 +294,15 @@ def species_initialize_inputs(self, layout, do_not_gather = self.do_not_gather, random_theta = self.random_theta, do_resampling=self.do_resampling, + resampling_algorithm=self.resampling_algorithm, + resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, - resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc) + resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, + resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, + resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, + resampling_algorithm_n_phi=self.resampling_algorithm_n_phi, + resampling_algorithm_delta_u=self.resampling_algorithm_delta_u) # add reflection models self.species.add_new_attr("reflection_model_xlo(E)", self.reflection_model_xlo) @@ -603,6 +648,11 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -630,6 +680,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -654,6 +707,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles pywarpx.boundary.reflect_all_velocities = self.reflect_all_velocities + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -707,6 +764,11 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -731,6 +793,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -747,6 +812,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -808,6 +877,11 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -834,6 +908,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -852,6 +929,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -929,6 +1010,11 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -957,6 +1043,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -977,6 +1066,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -1135,6 +1228,9 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): plasma_resistivity: float or str Value or expression to use for the plasma resistivity. + plasma_hyper_resistivity: float or str + Value or expression to use for the plasma hyper-resistivity. + substeps: int, default=100 Number of substeps to take when updating the B-field. @@ -1142,7 +1238,8 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): Function of space and time specifying external (non-plasma) currents. """ def __init__(self, grid, Te=None, n0=None, gamma=None, - n_floor=None, plasma_resistivity=None, substeps=None, + n_floor=None, plasma_resistivity=None, + plasma_hyper_resistivity=None, substeps=None, Jx_external_function=None, Jy_external_function=None, Jz_external_function=None, **kw): self.grid = grid @@ -1153,6 +1250,7 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.gamma = gamma self.n_floor = n_floor self.plasma_resistivity = plasma_resistivity + self.plasma_hyper_resistivity = plasma_hyper_resistivity self.substeps = substeps @@ -1187,6 +1285,7 @@ def solver_initialize_inputs(self): 'plasma_resistivity(rho,J)', pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) + pywarpx.hybridpicmodel.plasma_hyper_resistivity = self.plasma_hyper_resistivity pywarpx.hybridpicmodel.substeps = self.substeps pywarpx.hybridpicmodel.__setattr__( 'Jx_external_grid_function(x,y,z,t)', @@ -1678,7 +1777,6 @@ def embedded_boundary_initialize_inputs(self, solver): pywarpx.eb2.cover_multiple_cuts = self.cover_multiple_cuts if self.potential is not None: - assert isinstance(solver, ElectrostaticSolver), Exception('The potential is only supported with the ElectrostaticSolver') expression = pywarpx.my_constants.mangle_expression(self.potential, self.mangle_dict) pywarpx.warpx.__setattr__('eb_potential(x,y,z,t)', expression) @@ -1821,7 +1919,7 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_load_balance_knapsack_factor: float, default=1.24 (See documentation) - warpx_load_balance_costs_update: {'heuristic' or 'timers' or 'gpuclock'}, optional + warpx_load_balance_costs_update: {'heuristic' or 'timers'}, optional (See documentation) warpx_costs_heuristic_particles_wt: float, optional @@ -2246,11 +2344,13 @@ def diagnostic_initialize_inputs(self): E_fields_list = ['Er', 'Et', 'Ez'] B_fields_list = ['Br', 'Bt', 'Bz'] J_fields_list = ['Jr', 'Jt', 'Jz'] + J_displacement_fields_list = ['Jr_displacement', 'Jt_displacement', 'Jz_displacement'] A_fields_list = ['Ar', 'At', 'Az'] else: E_fields_list = ['Ex', 'Ey', 'Ez'] B_fields_list = ['Bx', 'By', 'Bz'] J_fields_list = ['Jx', 'Jy', 'Jz'] + J_displacement_fields_list = ['Jx_displacement', 'Jy_displacement', 'Jz_displacement'] A_fields_list = ['Ax', 'Ay', 'Az'] if self.data_list is not None: for dataname in self.data_list: @@ -2263,6 +2363,9 @@ def diagnostic_initialize_inputs(self): elif dataname == 'J': for field_name in J_fields_list: fields_to_plot.add(field_name.lower()) + elif dataname == 'J_displacement': + for field_name in J_displacement_fields_list: + fields_to_plot.add(field_name.lower()) elif dataname == 'A': for field_name in A_fields_list: fields_to_plot.add(field_name) @@ -2276,6 +2379,8 @@ def diagnostic_initialize_inputs(self): fields_to_plot.add(dataname) elif dataname in J_fields_list: fields_to_plot.add(dataname.lower()) + elif dataname in J_displacement_fields_list: + fields_to_plot.add(dataname.lower()) elif dataname.startswith('rho_'): # Adds rho_species diagnostic fields_to_plot.add(dataname) @@ -2687,47 +2792,47 @@ def diagnostic_initialize_inputs(self): # --- Use a set to ensure that fields don't get repeated. variables = set() - - for dataname in self.data_list: - if dataname == 'position': - if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z - variables.add('x') - if pywarpx.geometry.dims == '3': - variables.add('y') - variables.add('z') - if pywarpx.geometry.dims == 'RZ': - variables.add('theta') - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: - if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): - raise RuntimeError( - f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" - f"chosen by dim={pywarpx.geometry.dims} in pywarpx." - ) - elif pywarpx.geometry.dims != '3' and dataname == 'y': - raise RuntimeError( - f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" - f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." - ) - elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': - raise RuntimeError( - f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." - f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." - ) - else: - variables.add(dataname) + if self.data_list is not None: + for dataname in self.data_list: + if dataname == 'position': + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname == 'fields': + variables.add('Ex') + variables.add('Ey') + variables.add('Ez') + variables.add('Bx') + variables.add('By') + variables.add('Bz') + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. diff --git a/Python/setup.py b/Python/setup.py index 08e616cc279..fbf3330ada0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.03', + version = '24.05', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/README.md b/README.md index 150dbfea8ec..8ab69791bb1 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,6 @@ If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. -NOTICE. This Software was developed under funding from the U.S. Department -of Energy and the U.S. Government consequently retains certain rights. As -such, the U.S. Government has been granted for itself and others acting on -its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the -Software to reproduce, distribute copies to the public, prepare derivative -works, and perform publicly and display publicly, and to permit other to do -so. - Please see the full license agreement in [LICENSE.txt](LICENSE.txt). +Please see the notices in [NOTICE.txt](NOTICE.txt). The SPDX license identifier is `BSD-3-Clause-LBNL`. diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json index 693faf43882..799692135ce 100644 --- a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -1,96 +1,96 @@ { "lev=0": { - "Bx": 970573570349.2142, - "By": 970493693614.8823, - "Bz": 18743533.08373785, - "Ex": 2.9100804648817213e+20, - "Ey": 2.910188719414779e+20, - "Ez": 1.309938057448976e+17, - "rho_beam1": 7.969171294602906e+16, - "rho_beam2": 7.969079911431987e+16, - "rho_ele1": 234837475722889.97, - "rho_ele2": 277285508564124.12, - "rho_pos1": 229824415763789.62, - "rho_pos2": 286513388076434.4 + "Bx": 958613893612.4355, + "By": 958606286084.2804, + "Bz": 50291310.73170665, + "Ex": 2.873993867215236e+20, + "Ey": 2.8740263458334176e+20, + "Ez": 1.3469564521662371e+17, + "rho_beam1": 7.96926761425663e+16, + "rho_beam2": 7.969234189119718e+16, + "rho_ele1": 325562788515568.7, + "rho_ele2": 301373974746269.7, + "rho_pos1": 314013278169396.75, + "rho_pos2": 311298336003435.56 }, "beam1": { - "particle_opticalDepthQSR": 104947.58821047074, - "particle_position_x": 0.0015001846430437182, - "particle_position_y": 0.0015002200838688795, - "particle_position_z": 0.004965556992831605, - "particle_momentum_x": 6.207861637949496e-15, - "particle_momentum_y": 6.16591876835476e-15, - "particle_momentum_z": 6.806755306915448e-12, - "particle_weight": 635859587.7915188 + "particle_opticalDepthQSR": 104848.69019384333, + "particle_position_x": 0.0015001641452988207, + "particle_position_y": 0.0015001296473841738, + "particle_position_z": 0.004965480036291212, + "particle_momentum_x": 6.203657034942546e-15, + "particle_momentum_y": 6.161790111190829e-15, + "particle_momentum_z": 6.806194292286189e-12, + "particle_weight": 635868088.3867786 }, "beam2": { - "particle_opticalDepthQSR": 104180.42997257302, - "particle_position_x": 0.0015001132145418016, - "particle_position_y": 0.001500162338878405, - "particle_position_z": 0.004965528365042041, - "particle_momentum_x": 6.201791193788119e-15, - "particle_momentum_y": 6.188946480983165e-15, - "particle_momentum_z": 6.796967564632488e-12, - "particle_weight": 635853166.1373858 + "particle_opticalDepthQSR": 104197.28107371755, + "particle_position_x": 0.0015001452558405398, + "particle_position_y": 0.0015001281739351966, + "particle_position_z": 0.0049656445643994716, + "particle_momentum_x": 6.202758467582172e-15, + "particle_momentum_y": 6.18910011814166e-15, + "particle_momentum_z": 6.7994521022372906e-12, + "particle_weight": 635874794.3085052 }, "ele1": { - "particle_opticalDepthQSR": 385.3959944370225, - "particle_position_x": 5.063867198487091e-06, - "particle_position_y": 5.323267522706671e-06, - "particle_position_z": 1.81974337459613e-05, - "particle_momentum_x": 5.220808698440275e-18, - "particle_momentum_y": 5.2833645967924376e-18, - "particle_momentum_z": 2.4960852396311436e-15, - "particle_weight": 1883777.3071500752 + "particle_opticalDepthQSR": 398.7154177999122, + "particle_position_x": 5.2166787076833645e-06, + "particle_position_y": 5.005755590473258e-06, + "particle_position_z": 1.856829463647771e-05, + "particle_momentum_x": 6.0736878569270085e-18, + "particle_momentum_y": 5.735020185191748e-18, + "particle_momentum_z": 2.827581034346608e-15, + "particle_weight": 2602683.4209351614 }, "ele2": { - "particle_opticalDepthQSR": 391.0032239817431, - "particle_position_x": 5.442400321293304e-06, - "particle_position_y": 5.3621200149906786e-06, - "particle_position_z": 1.716649458876156e-05, - "particle_momentum_x": 4.7056270741663116e-18, - "particle_momentum_y": 4.414438514376292e-18, - "particle_momentum_z": 2.3816287305827113e-15, - "particle_weight": 2255238.1204111334 + "particle_opticalDepthQSR": 328.6975869797729, + "particle_position_x": 4.984003903707884e-06, + "particle_position_y": 4.695016970410262e-06, + "particle_position_z": 1.606918799511055e-05, + "particle_momentum_x": 4.524294388810778e-18, + "particle_momentum_y": 4.193609622515901e-18, + "particle_momentum_z": 2.624217472737641e-15, + "particle_weight": 2432495.8168380223 }, "pho1": { - "particle_opticalDepthBW": 9912.332305353064, - "particle_position_x": 0.0001424857999095925, - "particle_position_y": 0.00014366766674682975, - "particle_position_z": 0.0004764478273708734, + "particle_opticalDepthBW": 10028.214317531058, + "particle_position_x": 0.00014200324200040716, + "particle_position_y": 0.00014310262095706036, + "particle_position_z": 0.00047470309948487784, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61671336.44071695 + "particle_weight": 61455533.15171491 }, "pho2": { - "particle_opticalDepthBW": 10361.142481614075, - "particle_position_x": 0.00014718841339786984, - "particle_position_y": 0.00014538267635727008, - "particle_position_z": 0.00048768591004702896, + "particle_opticalDepthBW": 10261.48950301913, + "particle_position_x": 0.0001465092909391631, + "particle_position_y": 0.00014555115652303745, + "particle_position_z": 0.00048686081947093, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61889405.34553001 + "particle_weight": 61924991.09906147 }, "pos1": { - "particle_opticalDepthQSR": 343.0370605565725, - "particle_position_x": 5.41949101836001e-06, - "particle_position_y": 5.5292865311090835e-06, - "particle_position_z": 1.7063207973991194e-05, - "particle_momentum_x": 4.76704175252458e-18, - "particle_momentum_y": 5.049007427826035e-18, - "particle_momentum_z": 2.533578387785534e-15, - "particle_weight": 1821809.4309160584 + "particle_opticalDepthQSR": 380.4787933889546, + "particle_position_x": 5.59226140958729e-06, + "particle_position_y": 5.199149983019462e-06, + "particle_position_z": 1.7261766049926983e-05, + "particle_momentum_x": 5.182944941041321e-18, + "particle_momentum_y": 4.665394338329992e-18, + "particle_momentum_z": 2.565450485567441e-15, + "particle_weight": 2523696.1743423166 }, "pos2": { - "particle_opticalDepthQSR": 390.0211924195479, - "particle_position_x": 5.053169658257247e-06, - "particle_position_y": 5.039302796539821e-06, - "particle_position_z": 1.8526669235892217e-05, - "particle_momentum_x": 5.755216173987019e-18, - "particle_momentum_y": 5.056618594918483e-18, - "particle_momentum_z": 2.52324147594341e-15, - "particle_weight": 2328558.5523011778 + "particle_opticalDepthQSR": 378.7526306435402, + "particle_position_x": 4.812490588954386e-06, + "particle_position_y": 4.351750384371962e-06, + "particle_position_z": 1.7621416174292307e-05, + "particle_momentum_x": 4.979887438720444e-18, + "particle_momentum_y": 4.8215630209506066e-18, + "particle_momentum_z": 2.193964301475807e-15, + "particle_weight": 2513162.277112609 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json index 0d426c07ed9..76d03f521b8 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json @@ -1,4 +1,7 @@ { + "lev=0": { + "rho": 0.0 + }, "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -8,15 +11,6 @@ "particle_position_z": 81919021.52308556, "particle_weight": 1024.000000000021 }, - "helium3_1": { - "particle_momentum_x": 8.492383942294212e-16, - "particle_momentum_y": 8.507905928154269e-16, - "particle_momentum_z": 8.500197251270117e-16, - "particle_position_x": 151779.74563236872, - "particle_position_y": 152858.5501479658, - "particle_position_z": 322785.7613257161, - "particle_weight": 2.7065032055210726e-28 - }, "hydrogen2_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -26,16 +20,22 @@ "particle_position_z": 81920546.19181262, "particle_weight": 1024.000000000021 }, - "lev=0": { - "rho": 0.0 - }, "neutron_1": { - "particle_momentum_x": 8.492383942294212e-16, - "particle_momentum_y": 8.507905928154269e-16, - "particle_momentum_z": 8.500197251270117e-16, - "particle_position_x": 151779.74563236872, - "particle_position_y": 152858.5501479658, - "particle_position_z": 322785.7613257161, - "particle_weight": 2.7065032055210726e-28 + "particle_momentum_x": 8.574995119045322e-16, + "particle_momentum_y": 8.563358069072292e-16, + "particle_momentum_z": 8.571142005592407e-16, + "particle_position_x": 152851.7089469153, + "particle_position_y": 151779.17287974653, + "particle_position_z": 325950.1658699999, + "particle_weight": 2.7360848012854956e-28 + }, + "helium3_1": { + "particle_momentum_x": 8.574995119045322e-16, + "particle_momentum_y": 8.563358069072292e-16, + "particle_momentum_z": 8.571142005592407e-16, + "particle_position_x": 152851.7089469153, + "particle_position_y": 151779.17287974653, + "particle_position_z": 325950.1658699999, + "particle_weight": 2.7360848012854956e-28 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json index 1c6af40e173..59818f62826 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json @@ -1,4 +1,18 @@ { + "lev=0": { + "rho": 0.0, + "rho_deuterium": 8203144355.71195, + "rho_helium3": 10.368009592276463 + }, + "neutron": { + "particle_momentum_x": 2.2543499835759282e-15, + "particle_momentum_y": 2.2526527390783875e-15, + "particle_momentum_z": 2.2619641737859965e-15, + "particle_position_x": 61.961041864660686, + "particle_position_y": 61.78141653674165, + "particle_position_z": 61.741022731492514, + "particle_weight": 505562702.7678892 + }, "deuterium": { "particle_momentum_x": 1.3370046499332103e-14, "particle_momentum_y": 1.3364310231320824e-14, @@ -6,29 +20,15 @@ "particle_position_x": 2560.1613417364665, "particle_position_y": 2560.082464065988, "particle_position_z": 2560.0018477161034, - "particle_weight": 7.999999989895393e+17 + "particle_weight": 7.999999989888742e+17 }, "helium3": { - "particle_momentum_x": 2.2800935865848323e-15, - "particle_momentum_y": 2.2864983116288316e-15, - "particle_momentum_z": 2.290487090201156e-15, - "particle_position_x": 62.07326392385665, - "particle_position_y": 62.074468969610855, - "particle_position_z": 62.07087662970013, - "particle_weight": 505230602.2536906 - }, - "lev=0": { - "rho": 0.0, - "rho_deuterium": 8203144355.71876, - "rho_helium3": 10.36119892112141 - }, - "neutron": { - "particle_momentum_x": 2.2625878931428226e-15, - "particle_momentum_y": 2.2652096393691827e-15, - "particle_momentum_z": 2.27105060559148e-15, - "particle_position_x": 62.07326392385665, - "particle_position_y": 62.074468969610855, - "particle_position_z": 62.07087662970013, - "particle_weight": 505230602.2536906 + "particle_momentum_x": 2.2749239620327265e-15, + "particle_momentum_y": 2.268697031603961e-15, + "particle_momentum_z": 2.278045756364995e-15, + "particle_position_x": 61.961041864660686, + "particle_position_y": 61.78141653674165, + "particle_position_z": 61.741022731492514, + "particle_weight": 505562702.7678892 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json index 23231eab789..1ac85583f9d 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json @@ -2,76 +2,76 @@ "lev=0": { "rho": 0.0 }, - "neutron_1": { - "particle_momentum_x": 1.7519716491839538e-15, - "particle_momentum_y": 1.7523289312260283e-15, - "particle_momentum_z": 1.7480231586369996e-15, - "particle_position_x": 154379.32401483235, - "particle_position_y": 152618.63815943015, - "particle_position_z": 325970.4138010667, - "particle_weight": 4.421535775967805e-28 + "neutron_2": { + "particle_momentum_x": 1.5428541105411113e-15, + "particle_momentum_y": 1.5521530475782452e-15, + "particle_momentum_z": 1.5581061911002605e-15, + "particle_position_x": 139420.85239440785, + "particle_position_y": 137544.4080412421, + "particle_position_z": 292987.6112145397, + "particle_weight": 6.353652226479104e+18 }, - "tritium_1": { + "tritium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 + "particle_momentum_z": 0.0, + "particle_position_x": 409665.266470154, + "particle_position_y": 409535.84596852644, + "particle_position_z": 819126.8984535293, + "particle_weight": 1.0239999999364636e+29 }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.356220762633467e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503, - "particle_weight": 1.0240001137713503e+30 + "helium4_2": { + "particle_momentum_x": 1.5428541105411113e-15, + "particle_momentum_y": 1.5521530475782452e-15, + "particle_momentum_z": 1.7807114780425105e-15, + "particle_position_x": 139420.85239440785, + "particle_position_y": 137544.4080412421, + "particle_position_z": 292987.6112145397, + "particle_weight": 6.353652226479104e+18 }, - "helium4_1": { - "particle_momentum_x": 1.7519716491839538e-15, - "particle_momentum_y": 1.7523289312260283e-15, - "particle_momentum_z": 1.7480231586369996e-15, - "particle_position_x": 154379.32401483235, - "particle_position_y": 152618.63815943015, - "particle_position_z": 325970.4138010667, - "particle_weight": 4.421535775967805e-28 + "neutron_1": { + "particle_momentum_x": 1.7286463849660049e-15, + "particle_momentum_y": 1.7253692489143775e-15, + "particle_momentum_z": 1.7397872072929338e-15, + "particle_position_x": 151971.91531558792, + "particle_position_y": 150436.3752328273, + "particle_position_z": 320653.7895132215, + "particle_weight": 4.360021412453755e-28 }, "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, + "particle_position_x": 40958427.509923086, + "particle_position_y": 40959476.344507605, + "particle_position_z": 81921930.27522033, "particle_weight": 1024.000000000021 }, - "neutron_2": { - "particle_momentum_x": 1.538345593941914e-15, - "particle_momentum_y": 1.536969107959402e-15, - "particle_momentum_z": 1.5535605933245691e-15, - "particle_position_x": 137088.2335716813, - "particle_position_y": 136370.55273167047, - "particle_position_z": 290148.26756873244, - "particle_weight": 6.427760042646557e+18 - }, - "helium4_2": { - "particle_momentum_x": 1.538345593941914e-15, - "particle_momentum_y": 1.536969107959402e-15, - "particle_momentum_z": 1.769309377628039e-15, - "particle_position_x": 137088.2335716813, - "particle_position_y": 136370.55273167047, - "particle_position_z": 290148.26756873244, - "particle_weight": 6.427760042646557e+18 + "tritium_1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.8875978729147693e-13, + "particle_position_x": 40959200.1108159, + "particle_position_y": 40960650.40789148, + "particle_position_z": 81920772.79861207, + "particle_weight": 1024.000000000021 }, - "tritium_2": { + "deuterium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999357226e+29 + "particle_momentum_z": 3.356220762633467e-14, + "particle_position_x": 4096177.559084946, + "particle_position_y": 4096353.028787281, + "particle_position_z": 8192362.405430989, + "particle_weight": 1.024000113771424e+30 + }, + "helium4_1": { + "particle_momentum_x": 1.7286463849660049e-15, + "particle_momentum_y": 1.7253692489143775e-15, + "particle_momentum_z": 1.7397872072929338e-15, + "particle_position_x": 151971.91531558792, + "particle_position_y": 150436.3752328273, + "particle_position_z": 320653.7895132215, + "particle_weight": 4.360021412453755e-28 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json index 3fb0db8d8a5..bccb0a12b0f 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json @@ -1,64 +1,28 @@ { - "deuterium_1": { - "particle_momentum_x": 1.8388106511899905e-13, - "particle_momentum_y": 1.837868790009435e-13, - "particle_momentum_z": 0.0, - "particle_position_x": 40959919.499819286, - "particle_position_y": 81919224.48541151, - "particle_theta": 32166860.23003994, - "particle_weight": 3216.984554806547 - }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.336364094249911e-14, - "particle_position_x": 4095908.9083809257, - "particle_position_y": 8192069.080030457, - "particle_theta": 3216444.348910214, - "particle_weight": 3.1898417901971444e+29 - }, - "helium4_1": { - "particle_momentum_x": 1.858124399143442e-15, - "particle_momentum_y": 1.876715110797694e-15, - "particle_momentum_z": 1.7098432207359157e-15, - "particle_position_x": 152920.23233108618, - "particle_position_y": 323733.9138644398, - "particle_theta": 120064.13771707338, - "particle_weight": 1.603083276067953e-27 - }, - "helium4_2": { - "particle_momentum_x": 1.5195006688950936e-15, - "particle_momentum_y": 1.52430083815551e-15, - "particle_momentum_z": 1.7654865863613367e-15, - "particle_position_x": 136867.63803188328, - "particle_position_y": 286903.30393175944, - "particle_theta": 107912.20520382549, - "particle_weight": 2.0862696876352987e+19 - }, "lev=0": { "rho": 0.0 }, - "neutron_1": { - "particle_momentum_x": 1.7160671487712845e-15, - "particle_momentum_y": 1.7154753069055672e-15, - "particle_momentum_z": 1.7098432207359157e-15, - "particle_position_x": 152920.23233108618, - "particle_position_y": 323733.9138644398, - "particle_theta": 120064.13771707338, - "particle_weight": 1.603083276067953e-27 + "helium4_2": { + "particle_momentum_x": 1.500608194743691e-15, + "particle_momentum_y": 1.5109305701135078e-15, + "particle_momentum_z": 1.7349325969148392e-15, + "particle_position_x": 134553.8162997425, + "particle_position_y": 285988.79476631305, + "particle_theta": 106450.92963352974, + "particle_weight": 2.0493998275383095e+19 }, "neutron_2": { - "particle_momentum_x": 1.5195006688950936e-15, - "particle_momentum_y": 1.52430083815551e-15, - "particle_momentum_z": 1.5463311225724366e-15, - "particle_position_x": 136867.63803188328, - "particle_position_y": 286903.30393175944, - "particle_theta": 107912.20520382549, - "particle_weight": 2.0862696876352987e+19 + "particle_momentum_x": 1.500608194743691e-15, + "particle_momentum_y": 1.5109305701135078e-15, + "particle_momentum_z": 1.5208565520173107e-15, + "particle_position_x": 134553.8162997425, + "particle_position_y": 285988.79476631305, + "particle_theta": 106450.92963352974, + "particle_weight": 2.0493998275383095e+19 }, "tritium_1": { - "particle_momentum_x": 1.8384658063720362e-13, - "particle_momentum_y": 1.8381593257898129e-13, + "particle_momentum_x": 1.8384658063720355e-13, + "particle_momentum_y": 1.8381593257898126e-13, "particle_momentum_z": 0.0, "particle_position_x": 40961278.052658774, "particle_position_y": 81919046.8061561, @@ -72,6 +36,42 @@ "particle_position_x": 409793.9651940968, "particle_position_y": 819237.3558155322, "particle_theta": 321974.4557387621, - "particle_weight": 3.218514276139388e+29 + "particle_weight": 3.218514276143075e+29 + }, + "helium4_1": { + "particle_momentum_x": 1.8900919654269547e-15, + "particle_momentum_y": 1.88323376518843e-15, + "particle_momentum_z": 1.7117554385103313e-15, + "particle_position_x": 154816.5295407665, + "particle_position_y": 326401.3426220656, + "particle_theta": 121193.01462929095, + "particle_weight": 1.615304706407927e-27 + }, + "deuterium_1": { + "particle_momentum_x": 1.8388106511899905e-13, + "particle_momentum_y": 1.837868790009435e-13, + "particle_momentum_z": 0.0, + "particle_position_x": 40959919.499819286, + "particle_position_y": 81919224.48541151, + "particle_theta": 32166860.23003994, + "particle_weight": 3216.984554806547 + }, + "neutron_1": { + "particle_momentum_x": 1.7399029833189488e-15, + "particle_momentum_y": 1.7342017841614303e-15, + "particle_momentum_z": 1.7117554385103313e-15, + "particle_position_x": 154816.5295407665, + "particle_position_y": 326401.3426220656, + "particle_theta": 121193.01462929095, + "particle_weight": 1.615304706407927e-27 + }, + "deuterium_2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 3.336364094249911e-14, + "particle_position_x": 4095908.9083809257, + "particle_position_y": 8192069.080030457, + "particle_theta": 3216444.348910214, + "particle_weight": 3.18984179020083e+29 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json index 4e9bde52a78..45441acf466 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504803536004636, - "Ey": 6.504803536004637, - "Ez": 6.504803536004637, + "Ex": 6.504631859049922, + "Ey": 6.50463185904992, + "Ez": 6.504631859049919, "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 1.0084159184928337e-23, - "particle_momentum_y": 1.008415918492834e-23, - "particle_momentum_z": 1.0084159184928338e-23, - "particle_position_x": 518.418176976446, - "particle_position_y": 518.4181769764461, - "particle_position_z": 518.418176976446, + "particle_momentum_x": 9.882423407879068e-24, + "particle_momentum_y": 9.882423407879068e-24, + "particle_momentum_z": 9.882423407879066e-24, + "particle_position_x": 513.5229059338338, + "particle_position_y": 513.5229059338338, + "particle_position_z": 513.522905933834, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json new file mode 100644 index 00000000000..024127a1bd2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json @@ -0,0 +1,23 @@ +{ + "lev=0": { + "Ex": 7.110909624093144, + "Ey": 7.110909624093145, + "Ez": 7.110909624093143, + "rho": 0.0 + }, + "lev=1": { + "Ex": 14.281015560380963, + "Ey": 14.281015560380965, + "Ez": 14.281015560380965, + "rho": 2.6092568008333786e-10 + }, + "electron": { + "particle_momentum_x": 1.80842228672388e-24, + "particle_momentum_y": 1.8084222867238806e-24, + "particle_momentum_z": 1.7598771525647628e-24, + "particle_position_x": 327.46875, + "particle_position_y": 327.46875, + "particle_position_z": 327.46875, + "particle_weight": 6212.501525878906 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json index 8a79a510352..45441acf466 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json @@ -1,17 +1,17 @@ { - "electron": { - "particle_momentum_x": 1.0084159184928337e-23, - "particle_momentum_y": 1.008415918492834e-23, - "particle_momentum_z": 1.0084159184928338e-23, - "particle_position_x": 518.418176976446, - "particle_position_y": 518.4181769764461, - "particle_position_z": 518.418176976446, - "particle_weight": 6212.501525878906 - }, "lev=0": { - "Ex": 6.504803536004636, - "Ey": 6.504803536004637, - "Ez": 6.504803536004637, + "Ex": 6.504631859049922, + "Ey": 6.50463185904992, + "Ez": 6.504631859049919, "rho": 2.609256800833379e-10 + }, + "electron": { + "particle_momentum_x": 9.882423407879068e-24, + "particle_momentum_y": 9.882423407879068e-24, + "particle_momentum_z": 9.882423407879066e-24, + "particle_position_x": 513.5229059338338, + "particle_position_y": 513.5229059338338, + "particle_position_z": 513.522905933834, + "particle_weight": 6212.501525878906 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index 0efcdaeca3c..dd59536ad37 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 4818955.480792835, - "By": 1752.8025402207227, - "Bz": 14516.21278267981, - "Ex": 2366115496505.249, - "Ey": 1446112025634143.0, - "Ez": 21864189507353.19, - "jx": 1996366349839211.5, - "jy": 5.312583827165288e+16, - "jz": 2.049135262445976e+16, - "rho": 68443961.71835628 + "Bx": 4818955.480797943, + "By": 1752.8025638791275, + "Bz": 14516.212782554387, + "Ex": 2366115503598.9224, + "Ey": 1446112025635674.2, + "Ez": 21864189507357.867, + "jx": 1996366349775593.5, + "jy": 5.312583827155926e+16, + "jz": 2.0491352624508764e+16, + "rho": 68443961.71852128 }, "electrons": { - "particle_momentum_x": 2.2135945391227107e-23, - "particle_momentum_y": 2.8224559499572622e-22, - "particle_momentum_z": 5.260626010211241e-22, - "particle_position_x": 0.010800577787628053, + "particle_momentum_x": 2.2135945391319113e-23, + "particle_momentum_y": 2.8224559499558413e-22, + "particle_momentum_z": 5.260626010214114e-22, + "particle_position_x": 0.010800577787628052, "particle_position_y": 0.2111506062831815, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472277235318e-23, - "particle_momentum_y": 4.449097689427615e-22, - "particle_momentum_z": 5.768168724780326e-22, + "particle_momentum_x": 6.248472277246885e-23, + "particle_momentum_y": 4.449097689427654e-22, + "particle_momentum_z": 5.768168724998047e-22, "particle_position_x": 0.010800001678510512, "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 }, "beam": { - "particle_momentum_x": 3.535745635169933e-19, + "particle_momentum_x": 3.5357456351701565e-19, "particle_momentum_y": 4.363391839372122e-19, - "particle_momentum_z": 5.658606416951657e-17, - "particle_position_x": 0.008314723025211447, + "particle_momentum_z": 5.658606416951653e-17, + "particle_position_x": 0.008314723025211468, "particle_position_y": 1.1704335743854242, "particle_weight": 62415090744.60765 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index dd224516c5c..140bff16cc2 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497756265.44724107, - "By": 252197580.117894, - "Bz": 16988475.5444833, - "Ex": 7.299911104263803e+16, - "Ey": 1.460116488178734e+17, - "Ez": 2.26033100286114e+16, - "jx": 8.27644503757345e+17, - "jy": 2.1062078409959675e+18, - "jz": 2.8491727096438305e+19, - "rho": 91863764144.41415 - }, - "electrons": { - "particle_momentum_x": 5.76165226700654e-20, - "particle_momentum_y": 2.7567389504898156e-19, - "particle_momentum_z": 4.134562048099117e-19, - "particle_position_x": 0.0025269863605945427, - "particle_position_y": 0.0024538321295153346, - "particle_position_z": 0.17818421763751244, - "particle_weight": 1675789447169.5652 + "Bx": 499326907.0665159, + "By": 254256667.12630302, + "Bz": 16997157.572204895, + "Ex": 7.361789596479304e+16, + "Ey": 1.464987954436535e+17, + "Ez": 2.2588202735123868e+16, + "jx": 8.306469909158771e+17, + "jy": 2.1067947367705277e+18, + "jz": 2.86663445441587e+19, + "rho": 92447775221.08655 }, "beam": { - "particle_momentum_x": 1.1926313055043134e-17, - "particle_momentum_y": 2.7056205218547404e-17, - "particle_momentum_z": 5.1562131494813424e-14, - "particle_position_x": 0.000560821518739447, - "particle_position_y": 0.000800729816549036, - "particle_position_z": 2.9800048650570097, + "particle_momentum_x": 1.1926337276935442e-17, + "particle_momentum_y": 2.70596966890012e-17, + "particle_momentum_z": 5.1562142636293934e-14, + "particle_position_x": 0.0005608203524288058, + "particle_position_y": 0.000800752801704984, + "particle_position_z": 2.980004865036241, "particle_weight": 62415.090744607616 + }, + "electrons": { + "particle_momentum_x": 5.774426971193643e-20, + "particle_momentum_y": 2.742475325800203e-19, + "particle_momentum_z": 4.1461064946749695e-19, + "particle_position_x": 0.0025250307389502184, + "particle_position_y": 0.0024550212004677482, + "particle_position_z": 0.17819223528078157, + "particle_weight": 1675789447169.5652 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json index 30eef0cedb7..90493698df0 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json @@ -22,4 +22,4 @@ "particle_momentum_z": 4.231730764749506e-20, "particle_weight": 12926557617.187498 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index 242cf00f0c3..379e0991819 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 100278645.72225758, - "Bz": 2509008.1146699786, - "Er": 3.263343388037509, - "Et": 3.0045297157982412e+16, - "Ez": 0.2422526411295923, - "jr": 49.67802097760541, - "jt": 4.3051397628943917e+17, + "Br": 100277153.12954171, + "Bz": 2508844.115818364, + "Er": 3.263693682174799, + "Et": 3.004484377567921e+16, + "Ez": 0.24219246674343975, + "jr": 49.68134418425658, + "jt": 4.305088291175302e+17, "jz": 0.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json index 8d58f5c9551..ef53dcabeec 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 193609031.89343664, - "Bz": 4689634.787639907, - "Er": 197280940689075.1, - "Et": 5.9716806429474664e+16, - "Ez": 1383812184076636.0, - "jr": 18582087468.75, - "jt": 1.193738117372394e+18, + "Br": 193606866.6957581, + "Bz": 4689375.7330573285, + "Er": 197284500110989.84, + "Et": 5.971613843782555e+16, + "Ez": 1383808350045244.5, + "jr": 18582922265.875, + "jt": 1.193688996955741e+18, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json b/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json index 2c60799425a..0692546f6cf 100644 --- a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json +++ b/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json @@ -1,15 +1,15 @@ { "lev=0": { - "Bx": 4966241349161.254, - "By": 4966241349161.246, - "Bz": 6.831134328962481e-11, - "Ex": 1.4888417010888241e+21, - "Ey": 1.4888417010888241e+21, - "Ez": 8138626456.10672, - "g_beam_p": 175096250199.0888, + "Bx": 4964230908893.931, + "By": 4964230908893.93, + "Bz": 7.391430009205359e-11, + "Ex": 1.4882389862594233e+21, + "Ey": 1.4882389862594225e+21, + "Ez": 8130823640.561069, + "g_beam_p": 175096250199.08884, "rho_beam_p": 8.331476384415223e+17, - "vx_beam_p": 3.9150198762467515e-10, - "vy_beam_p": 3.915019770559888e-10, + "vx_beam_p": 3.912093052231238e-10, + "vy_beam_p": 3.9120930706063816e-10, "vz_beam_p": 715792.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index 7f359f76e94..cdee4b078b0 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -3,115 +3,115 @@ "rho": 0.0 }, "alpha2": { - "particle_momentum_x": 4.0815251136381854e-15, - "particle_momentum_y": 4.122335955939925e-15, - "particle_momentum_z": 4.201726051973563e-15, - "particle_position_x": 410664.0477768091, - "particle_position_y": 868193.1483606595, - "particle_weight": 2.947401325355292e+18 + "particle_momentum_x": 4.057984510682467e-15, + "particle_momentum_y": 4.104188139725855e-15, + "particle_momentum_z": 4.17858000090827e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 5.078061191951185e+18 }, "alpha3": { - "particle_momentum_x": 4.925995964082186e-16, - "particle_momentum_y": 4.780894181553476e-16, - "particle_momentum_z": 4.688680302272935e-16, - "particle_position_x": 52240.870920562535, - "particle_position_y": 103873.30862578771, - "particle_weight": 1.4784234459335885e+27 + "particle_momentum_x": 5.017656304003558e-16, + "particle_momentum_y": 4.935595075276182e-16, + "particle_momentum_z": 4.867133212376827e-16, + "particle_position_x": 52678.192400911765, + "particle_position_y": 105483.59950020742, + "particle_weight": 1.5413633830148085e+27 }, - "proton1": { + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, "particle_weight": 128.00000000000261 }, - "alpha4": { - "particle_momentum_x": 2.3422553081132076e-14, - "particle_momentum_y": 2.3416860254545166e-14, - "particle_momentum_z": 2.3532380279126124e-14, - "particle_position_x": 2457367.4582781536, - "particle_position_y": 4915112.044373058, - "particle_weight": 384.0000000000002 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399507192184686e+30 - }, "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, "particle_position_x": 409798.015821768, "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999999017534e+28 + "particle_weight": 1.2799999998307316e+28 }, - "boron1": { + "boron5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 }, - "proton3": { - "particle_momentum_x": 1.684673285246867e-15, - "particle_momentum_y": 1.6827557106531144e-15, - "particle_momentum_z": 1.6802642612723895e-15, - "particle_position_x": 2457259.537951346, - "particle_position_y": 4914248.771185745, - "particle_weight": 1.2795071921846893e+30 + "alpha1": { + "particle_momentum_x": 4.691909092431811e-15, + "particle_momentum_y": 4.6836958163275755e-15, + "particle_momentum_z": 4.657203546376977e-15, + "particle_position_x": 463465.27131109464, + "particle_position_y": 978207.6061359186, + "particle_weight": 4.977661460251435e-28 }, - "proton5": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512788807322e+28 }, - "alpha1": { - "particle_momentum_x": 4.714228774936194e-15, - "particle_momentum_y": 4.676079523757908e-15, - "particle_momentum_z": 4.676143086393951e-15, - "particle_position_x": 463834.9497057527, - "particle_position_y": 977592.6452214213, - "particle_weight": 2.9280275175431764e-28 + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, + "particle_weight": 128.00000000000261 }, "alpha5": { - "particle_momentum_x": 2.3300446729308862e-14, - "particle_momentum_y": 2.3449941485701153e-14, - "particle_momentum_z": 2.3618372938672865e-14, - "particle_position_x": 2457556.8571638414, - "particle_position_y": 4914659.635379325, + "particle_momentum_x": 2.3343485859070736e-14, + "particle_momentum_y": 2.3451128753701046e-14, + "particle_momentum_z": 2.3579462789062662e-14, + "particle_position_x": 2457556.8571638423, + "particle_position_y": 4914659.635379322, "particle_weight": 3.839999999999998e-19 }, - "proton2": { + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399486212205656e+30 + }, + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789517532e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 }, - "boron5": { + "proton3": { + "particle_momentum_x": 1.6847290893972251e-15, + "particle_momentum_y": 1.6827074502304075e-15, + "particle_momentum_z": 1.6802489646490975e-15, + "particle_position_x": 2457270.6999197667, + "particle_position_y": 4914315.665267942, + "particle_weight": 1.279486212205663e+30 + }, + "alpha4": { + "particle_momentum_x": 2.338084461204216e-14, + "particle_momentum_y": 2.3436156778849828e-14, + "particle_momentum_z": 2.35535708386288e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 4915112.044373056, + "particle_weight": 384.0000000000002 + }, + "proton5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index c3517c32b5b..ec7e047c537 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,131 +1,131 @@ { "lev=0": { - "rho": 0.0 + "rho": 0.0 }, - "proton3": { - "particle_momentum_x": 1.684809640261926e-15, - "particle_momentum_y": 1.6828399494797829e-15, - "particle_momentum_z": 1.6804535487104095e-15, - "particle_position_x": 2457338.899376694, - "particle_position_y": 2457069.647393952, - "particle_position_z": 4914642.288898885, - "particle_weight": 1.0236580897818594e+31 - }, - "alpha5": { - "particle_momentum_x": 2.3336421176154494e-14, - "particle_momentum_y": 2.345583787205298e-14, - "particle_momentum_z": 2.3574859187827772e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 2457059.6353793233, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119658089781855e+31 + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 }, "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "alpha4": { - "particle_momentum_x": 2.3374367002622734e-14, - "particle_momentum_y": 2.3439200330576348e-14, - "particle_momentum_z": 2.3559200621107925e-14, - "particle_position_x": 2457367.4582781536, - "particle_position_y": 2457512.044373057, - "particle_position_z": 4914475.776513074, - "particle_weight": 3072.000000000002 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 }, "alpha3": { - "particle_momentum_x": 4.857656981301579e-16, - "particle_momentum_y": 4.811177053359803e-16, - "particle_momentum_z": 4.766749863500389e-16, - "particle_position_x": 51282.9136300544, - "particle_position_y": 51045.39533309579, - "particle_position_z": 101578.82008857065, - "particle_weight": 1.0257306544231767e+28 + "particle_momentum_x": 4.764404554793872e-16, + "particle_momentum_y": 4.655900875811434e-16, + "particle_momentum_z": 4.578927372510084e-16, + "particle_position_x": 50987.442011759704, + "particle_position_y": 48999.674675246955, + "particle_position_z": 101142.57224226737, + "particle_weight": 1.0633705344227063e+28 }, - "alpha2": { - "particle_momentum_x": 4.071664828713061e-15, - "particle_momentum_y": 4.119627260272026e-15, - "particle_momentum_z": 4.181812341443065e-15, - "particle_position_x": 405848.1200755021, - "particle_position_y": 408011.53191288817, - "particle_position_z": 866330.5923068221, - "particle_weight": 1.9180757241165136e+19 + "alpha1": { + "particle_momentum_x": 4.665933695243743e-15, + "particle_momentum_y": 4.603805875733438e-15, + "particle_momentum_z": 4.706765986105302e-15, + "particle_position_x": 461871.79172011977, + "particle_position_y": 461162.2166206925, + "particle_position_z": 969262.7809050508, + "particle_weight": 3.2387855108185994e-27 + }, + "alpha5": { + "particle_momentum_x": 2.3388206254864998e-14, + "particle_momentum_y": 2.334372885765467e-14, + "particle_momentum_z": 2.363588638941874e-14, + "particle_position_x": 2457556.857163843, + "particle_position_y": 2457059.6353793247, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 }, "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3745333755307162e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240716198e+29 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.022781024024315e+29 }, "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.1196455431551905e+31 }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 + "alpha4": { + "particle_momentum_x": 2.33898275612641e-14, + "particle_momentum_y": 2.3423797451957437e-14, + "particle_momentum_z": 2.3516107929259732e-14, + "particle_position_x": 2457367.458278153, + "particle_position_y": 2457512.0443730573, + "particle_position_z": 4914475.7765130745, + "particle_weight": 3072.000000000002 }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 + "proton3": { + "particle_momentum_x": 1.6847282386883186e-15, + "particle_momentum_y": 1.6828065767793222e-15, + "particle_momentum_z": 1.6803456707569493e-15, + "particle_position_x": 2457343.371083716, + "particle_position_y": 2457033.3891170574, + "particle_position_z": 4914529.855222688, + "particle_weight": 1.023645543155193e+31 }, - "alpha1": { - "particle_momentum_x": 4.659632773478301e-15, - "particle_momentum_y": 4.617309192789389e-15, - "particle_momentum_z": 4.6173901462667306e-15, - "particle_position_x": 453219.3436004627, - "particle_position_y": 457393.42114332493, - "particle_position_z": 970481.0580153911, - "particle_weight": 1.8966442168729786e-27 + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 }, "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.023999999936064e+29 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.0239999998887592e+29 + }, + "alpha2": { + "particle_momentum_x": 4.1179548991012315e-15, + "particle_momentum_y": 4.110026665992801e-15, + "particle_momentum_z": 4.169802553223462e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 3.3372246639840338e+19 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json index 0b38f78e6f9..53c3708056f 100644 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "rho_electrons": 0.00443743609125863, - "rho_he_ions": 0.005198801518328451 + "rho_electrons": 0.004434311371276535, + "rho_he_ions": 0.005200489146365628 + }, + "he_ions": { + "particle_momentum_x": 2.7688639005236794e-19, + "particle_momentum_y": 2.760648864116014e-19, + "particle_momentum_z": 3.6226628630061563e-19, + "particle_position_x": 2201.614485497906, + "particle_weight": 17190996093750.002 }, "neutrals": { - "particle_momentum_x": 1.404700281648976e-19, - "particle_momentum_y": 1.4028127127618884e-19, - "particle_momentum_z": 1.4090901433394346e-19, - "particle_position_x": 1120.7727446759352, + "particle_momentum_x": 1.4040775167811838e-19, + "particle_momentum_y": 1.4009514702703257e-19, + "particle_momentum_z": 1.4093144247152345e-19, + "particle_position_x": 1119.524782452131, "particle_weight": 6.4588e+19 }, - "he_ions": { - "particle_momentum_x": 2.770386771117138e-19, - "particle_momentum_y": 2.7568040242914223e-19, - "particle_momentum_z": 3.619756966185903e-19, - "particle_position_x": 2200.683185473434, - "particle_weight": 17185500000000.002 - }, "electrons": { - "particle_momentum_x": 3.5129762363657864e-20, - "particle_momentum_y": 3.5431134517510143e-20, - "particle_momentum_z": 1.2592093336142964e-19, - "particle_position_x": 2142.0662480700303, - "particle_weight": 14593699218750.002 + "particle_momentum_x": 3.5193566020597954e-20, + "particle_momentum_y": 3.5368803263788353e-20, + "particle_momentum_z": 1.2572273121326582e-19, + "particle_position_x": 2139.3709122461873, + "particle_weight": 14579566406250.002 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json index f5606cfee2f..5e42d7ec14b 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json @@ -5,7 +5,10 @@ "Bz": 256.0, "Ex": 1954.3267519602405, "Ey": 2363.4281756166347, - "Ez": 4873.508158589938 + "Ez": 4873.508158589938, + "jx_displacement": 914396602.5344071, + "jy_displacement": 762623912.4748704, + "jz_displacement": 2938492698.375407 }, "ions": { "particle_momentum_x": 1.6151135948675135e-19, @@ -15,3 +18,4 @@ "particle_weight": 4.220251350277737e+21 } } + diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json index ca50554c394..ec1b6272092 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json @@ -1,12 +1,12 @@ { "lev=0": {}, "ions": { - "particle_momentum_x": 5.043858996017125e-17, - "particle_momentum_y": 5.0444743275098145e-17, - "particle_momentum_z": 5.05192925609973e-17, - "particle_position_x": 143164.42694236583, - "particle_position_y": 143166.51848290052, - "particle_theta": 2573261.754119082, - "particle_weight": 8.12868064536689e+18 + "particle_momentum_x": 5.0438993756415296e-17, + "particle_momentum_y": 5.0444406612873916e-17, + "particle_momentum_z": 5.0519292431385393e-17, + "particle_position_x": 143164.41713467025, + "particle_position_y": 143166.51845281923, + "particle_theta": 2573261.8729711357, + "particle_weight": 8.128680645366887e+18 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json index c7b467ad5f4..316bcb5b30d 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json @@ -2,17 +2,17 @@ "lev=0": { "Bx": 1524.8789585554075, "By": 639.8314135126764, - "Bz": 7.476000395458839, - "Ex": 56499935.20497879, - "Ey": 75068107.08471295, - "Ez": 309535670.18599623 + "Bz": 7.476064371022787, + "Ex": 56500024.54347144, + "Ey": 75064104.7364582, + "Ez": 309548482.989921 }, "ions": { - "particle_momentum_x": 7.14295522755638e-15, - "particle_momentum_y": 7.1380780610425e-15, - "particle_momentum_z": 7.141134469227045e-15, - "particle_position_x": 11170689.203149632, - "particle_position_y": 5585328.083196239, + "particle_momentum_x": 7.142955224072594e-15, + "particle_momentum_y": 7.13807805985974e-15, + "particle_momentum_z": 7.141134511815448e-15, + "particle_position_x": 11170689.202853566, + "particle_position_y": 5585328.08326772, "particle_weight": 9.036667901693183e+18 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_prev_positions.json b/Regression/Checksum/benchmarks_json/Python_prev_positions.json index 55c886775d5..8c7c6bf8b73 100644 --- a/Regression/Checksum/benchmarks_json/Python_prev_positions.json +++ b/Regression/Checksum/benchmarks_json/Python_prev_positions.json @@ -3,21 +3,21 @@ "Bx": 0.0, "By": 0.0, "Bz": 0.0, - "Ex": 3710588.849989976, + "Ex": 3710700.956616212, "Ey": 0.0, - "Ez": 646727.8074440088, - "jx": 15259.034603501308, - "jy": 650.139263398662, - "jz": 943.0244062246846 + "Ez": 646752.4642948855, + "jx": 15258.82611558372, + "jy": 650.13507310447, + "jz": 942.8394748261528 }, "electrons": { - "particle_momentum_x": 8.78764082600202e-23, + "particle_momentum_x": 8.787556176835159e-23, "particle_momentum_y": 3.78128494181519e-24, - "particle_momentum_z": 5.3969075504746586e-24, - "particle_position_x": 0.6227948811898449, - "particle_position_y": 0.5553787034074176, - "particle_prev_x": 0.6152547961491118, - "particle_prev_z": 0.5559363018627721, + "particle_momentum_z": 5.395758515337991e-24, + "particle_position_x": 0.6227736757162666, + "particle_position_y": 0.5553800210930071, + "particle_prev_x": 0.615237399194202, + "particle_prev_z": 0.5559373934585523, "particle_weight": 8569335937.5 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_restart_eb.json b/Regression/Checksum/benchmarks_json/Python_restart_eb.json index 25087c5757d..ad0d2cee5a3 100644 --- a/Regression/Checksum/benchmarks_json/Python_restart_eb.json +++ b/Regression/Checksum/benchmarks_json/Python_restart_eb.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 148618.63186220315, - "By": 148618.63186220315, - "Bz": 3385.851454453729, - "Ex": 55362729623335.9, - "Ey": 55362729623335.89, - "Ez": 68396725892689.86 + "Bx": 148673.005859208, + "By": 148673.00585920806, + "Bz": 3371.758117878558, + "Ex": 55378581103426.71, + "Ey": 55378581103426.72, + "Ez": 68412803445328.25 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 742cb775230..96a2f5cbdac 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -1,13 +1,4 @@ { - "electron": { - "particle_momentum_x": 3.5796522874263304e-19, - "particle_momentum_y": 3.580087349086981e-19, - "particle_momentum_z": 3.5805012694309815e-19, - "particle_position_x": 1.024116253197213, - "particle_position_y": 1.0238635904620974, - "particle_position_z": 1.0240133505121767, - "particle_weight": 714240000000.0 - }, "lev=0": { "Bx": 0.0, "By": 0.0, @@ -18,5 +9,14 @@ "jx": 0.0, "jy": 0.0, "jz": 0.0 + }, + "electron": { + "particle_momentum_x": 3.5856774369040155e-19, + "particle_momentum_y": 3.5832349891760786e-19, + "particle_momentum_z": 3.5757013442445486e-19, + "particle_position_x": 1.0241562532152744, + "particle_position_y": 1.0238555904782611, + "particle_position_z": 1.0240053504978093, + "particle_weight": 714240000000.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json index 69fffed2c6a..d0edafb9f0e 100644 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json +++ b/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json @@ -1,11 +1,10 @@ { "lev=0": { - "Bx": 0.0, - "By": 0.0066283741197868335, - "Bz": 0.006628374119786833, - "Ex": 5102618.47115243, - "Ey": 0.0, - "Ez": 0.0 + "Bx": 3.769898030127477e-18, + "By": 0.006628374119786834, + "Bz": 0.006628374119786834, + "Ex": 5102618.4711524295, + "Ey": 6.323755130400527e-05, + "Ez": 6.323755130400527e-05 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json index 37a82967a23..a3e609bd9a9 100644 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json +++ b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 9.263694545408502e-05, - "By": 0.00031905198933489135, - "Bz": 7.328424783762596e-05, - "Ex": 8553.90669805307, + "Bx": 9.263694545408503e-05, + "By": 0.00031905198933489145, + "Bz": 7.328424783762594e-05, + "Ex": 8553.906698053046, "Ey": 60867.04830538045, - "Ez": 0.0 + "Ez": 8.439422682267567e-07 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json index 453b83f5e31..c759f36ac5d 100644 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json +++ b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 0.0, - "By": 0.005101824310293575, + "Bx": 3.898540288712651e-18, + "By": 0.005101824310293573, "Bz": 0.005101824310293573, - "Ex": 4414725.184731115, - "Ey": 0.0, - "Ez": 0.0 + "Ex": 4414725.184731114, + "Ey": 6.323754812712063e-05, + "Ez": 6.323754812712063e-05 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json index f8186748f2c..8a1ff9de966 100644 --- a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json +++ b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json @@ -3,20 +3,20 @@ "Bx": 0.0, "By": 0.0, "Bz": 0.0, - "Ex": 9.882421146615367e-06, - "Ey": 1.0440261046714249e-05, - "Ez": 1.003739697324731e-05, - "jx": 2.914866280957325e-10, - "jy": 8.46605718473121e-19, - "jz": 3.82349275686397e-08 + "Ex": 8.263130364346929e-06, + "Ey": 8.74492138450055e-06, + "Ez": 8.97527046111985e-06, + "jx": 2.9148662813789523e-10, + "jy": 8.513426949250112e-19, + "jz": 3.8234927568748794e-08 }, "electron": { - "particle_momentum_x": 2.0819392991319055e-25, - "particle_momentum_y": 6.046869894766189e-34, - "particle_momentum_z": 2.730924530729043e-23, - "particle_position_x": 0.03492328774658799, - "particle_position_y": 2.2742551618036812e-11, - "particle_position_z": 1.4915217664612082, + "particle_momentum_x": 2.0819392994331126e-25, + "particle_momentum_y": 6.080703685011498e-34, + "particle_momentum_z": 2.7309245307369126e-23, + "particle_position_x": 0.03492328774773231, + "particle_position_y": 2.3430381516062662e-11, + "particle_position_z": 1.491521766466965, "particle_weight": 1.0 } } diff --git a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json index 801611f748b..a1ec0b4c831 100644 --- a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json +++ b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json @@ -1,21 +1,21 @@ { - "beam": { - "particle_momentum_x": 1.356321653466651e-21, - "particle_momentum_y": 1.356321653466648e-21, - "particle_momentum_z": 7.150873477755126e-16, - "particle_position_x": 11163.999973966627, - "particle_position_y": 11163.999973966625, - "particle_position_z": 131662.5003103599, - "particle_weight": 20895107655113.465 - }, "lev=0": { - "Az": 11.35866332708226, - "Bx": 111.55929404955174, - "By": 111.55929404955177, - "Ex": 31268316751.31371, - "Ey": 31268316751.31371, - "jz": 1034841326.0482643, - "phi": 3144575598.841474, + "Az": 11.358663326449284, + "Bx": 111.55929407644248, + "By": 111.55929407644244, + "Ex": 31257180402.55472, + "Ey": 31257180402.55473, + "jz": 1034841325.9848926, + "phi": 3143521213.0157924, "rho": 3.449203918900721 + }, + "beam": { + "particle_momentum_x": 1.3604657334742729e-21, + "particle_momentum_y": 1.3604657334742772e-21, + "particle_momentum_z": 7.150873450281544e-16, + "particle_position_x": 11163.99997371537, + "particle_position_y": 11163.999973715368, + "particle_position_z": 131662.50031035842, + "particle_weight": 20895107655113.465 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json new file mode 100644 index 00000000000..e4ff1fc68a8 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 100915933.44993827, + "By": 157610622.1855512, + "Bz": 9.717358898362187e-14, + "Ex": 4.7250652706211096e+16, + "Ey": 3.0253948990559976e+16, + "Ez": 3276573.9514776524, + "rho": 10994013582437.193 + }, + "electron": { + "particle_momentum_x": 5.701277606050295e-19, + "particle_momentum_y": 3.6504516641520437e-19, + "particle_momentum_z": 1.145432768297242e-10, + "particle_position_x": 17.314086912497864, + "particle_position_y": 0.2583691267187796, + "particle_position_z": 10066.329600000008, + "particle_weight": 19969036501.910976 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json new file mode 100644 index 00000000000..318aab0a1bc --- /dev/null +++ b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json @@ -0,0 +1,12 @@ +{ + "lev=0": { + "Bx": 153.42850019293041, + "By": 290.08173598472, + "Bz": 187.69788476927954, + "Ex": 703014828960.0149, + "Ey": 28192823505.416325, + "Ez": 693698719737.6497, + "divE": 4.868580396562553e+19, + "rho": 324055400.4292488 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json b/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json index 2944a6ae69a..d76d1ed7d4c 100644 --- a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json +++ b/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json @@ -1,27 +1,27 @@ { + "lev=0": { + "Bx": 0.08424138754164545, + "By": 0.08475193383267765, + "Bz": 0.08796438426563924, + "Ex": 106872541.64261818, + "Ey": 107805199.35060501, + "Ez": 107782783.64583969, + "jx": 728219.6885312796, + "jy": 736402.7559853494, + "jz": 734812.8777186275, + "rho": 0.02775080584090528, + "rho_electrons": 0.5250012277583664, + "rho_protons": 0.5250012273027096 + }, "electrons": { - "particle_momentum_x": 2.4417210113801336e-19, - "particle_momentum_y": 2.4564851493412593e-19, - "particle_momentum_z": 2.4341594564719795e-19, - "particle_position_x": 16379.247754118871, - "particle_position_y": 16382.836191849463, - "particle_position_z": 16382.833818260406, + "particle_momentum_x": 2.4417210857855684e-19, + "particle_momentum_y": 2.4564845792611e-19, + "particle_momentum_z": 2.434159547126453e-19, + "particle_position_x": 16379.247745715009, + "particle_position_y": 16382.83609924704, + "particle_position_z": 16382.833801764602, "particle_weight": 800000003014656.0 }, - "lev=0": { - "Bx": 0.084241235376183, - "By": 0.084751961809296, - "Bz": 0.08796431812220362, - "Ex": 106872513.1172905, - "Ey": 107805221.85876846, - "Ez": 107782819.9927826, - "jx": 728219.9166720062, - "jy": 736402.6373479366, - "jz": 734813.3440347463, - "rho": 0.0277508111536644, - "rho_electrons": 0.52500122784204, - "rho_protons": 0.5250012273481843 - }, "photons": { "particle_momentum_x": 1.428291590249666e-18, "particle_momentum_y": 1.4222174024686332e-18, @@ -32,12 +32,12 @@ "particle_weight": 800000003014656.0 }, "protons": { - "particle_momentum_x": 1.4104799077996007e-19, - "particle_momentum_y": 1.4120357666424943e-19, - "particle_momentum_z": 1.3903168923764969e-19, - "particle_position_x": 16383.951009266078, - "particle_position_y": 16383.991230053827, - "particle_position_z": 16384.033095981926, + "particle_momentum_x": 1.4104797829545176e-19, + "particle_momentum_y": 1.4120353509003028e-19, + "particle_momentum_z": 1.3903169103008315e-19, + "particle_position_x": 16383.951009267941, + "particle_position_y": 16383.991229955107, + "particle_position_z": 16384.03309576772, "particle_weight": 800000003014656.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json b/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json index d97f8b58053..adb4477378b 100644 --- a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json +++ b/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json @@ -1,11 +1,11 @@ { "lev=0": { - "By": 3.176136185686860e-05, - "Ex": 9.522292819119155e+03, - "Ey": 9.516582886012895e+03, - "Ez": 8.118976924626002, + "By": 0.00025366943932442746, + "Ex": 76051.98704866154, + "Ey": 76009.35086584378, + "Ez": 64.90079149285842, "jx": 0.0, "jy": 0.0, - "jz": 785848.6496458045 + "jz": 6286789.197166436 } } diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json new file mode 100644 index 00000000000..1f6f076f021 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 447073.7680171198, + "jy": 336566.4197455162, + "jz": 642997.1912520501 + }, + "hydrogen": { + "particle_momentum_x": 1.7562033925112653e-17, + "particle_momentum_y": 1.756681695333869e-17, + "particle_momentum_z": 2.0662018455459364e-17, + "particle_position_x": 6600.315303519964, + "particle_weight": 1.0000000000000003e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json new file mode 100644 index 00000000000..9f1dae16e7f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 434240.88525516074, + "jy": 315768.9402331213, + "jz": 642846.5589763735 + }, + "hydrogen": { + "particle_momentum_x": 6.919253235905925e-20, + "particle_momentum_y": 7.069867215063566e-20, + "particle_momentum_z": 7.606884436016626e-20, + "particle_position_x": 25.60006676227001, + "particle_weight": 1.0000000000000001e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization.json b/Regression/Checksum/benchmarks_json/space_charge_initialization.json index e0a157f2a81..07916316a06 100644 --- a/Regression/Checksum/benchmarks_json/space_charge_initialization.json +++ b/Regression/Checksum/benchmarks_json/space_charge_initialization.json @@ -1,8 +1,8 @@ { "lev=0": { - "Ex": 7869.6596181638515, - "Ey": 7863.981043503829, - "Ez": 7869.780361756294, + "Ex": 62838.484914953624, + "Ey": 62796.0798671538, + "Ez": 62839.82091026324, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json b/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json index f37add38914..6ca98880d4a 100644 --- a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json +++ b/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json @@ -1,8 +1,8 @@ { "lev=0": { - "Ex": 7438.125125466449, + "Ex": 29702.801592163793, "Ey": 0.0, - "Ez": 7438.329203373256, + "Ez": 29704.478300339702, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/spacecraft_charging.json index d9f753e1df2..2994d4feeae 100644 --- a/Regression/Checksum/benchmarks_json/spacecraft_charging.json +++ b/Regression/Checksum/benchmarks_json/spacecraft_charging.json @@ -1,30 +1,28 @@ { "lev=0": { - "Er": 75713.05000099652, - "Ez": 75260.78239853957, - "phi": 55650.30604185804, - "rho": 1.4793075271598396e-06, - "rho_electrons": 6.506538129003745e-06, - "rho_protons": 6.98347902659172e-06 + "Er": 72926.92774219153, + "Ez": 71877.43945667242, + "phi": 58270.32861572235, + "rho": 1.460951374255109e-06, + "rho_electrons": 6.501141119196457e-06, + "rho_protons": 6.992817056027598e-06 }, "electrons": { - "particle_position_x": 38158.7364935527, - "particle_position_y": 37779.25499255196, - "particle_position_z": 45010.371467374425, - "particle_momentum_x": 8.27307207197173e-20, - "particle_momentum_y": 8.264475255806164e-20, - "particle_momentum_z": 8.271327169054914e-20, - "particle_weight": 1140673608016.2212 + "particle_position_x": 38094.07563382932, + "particle_position_y": 37781.48683071032, + "particle_position_z": 44952.3745721341, + "particle_momentum_x": 8.278426203619687e-20, + "particle_momentum_y": 8.272126720129096e-20, + "particle_momentum_z": 8.268559239607837e-20, + "particle_weight": 1140843527312.664 }, "protons": { - "particle_position_x": 751407.372588289, - "particle_position_y": 751687.788498272, - "particle_position_z": 644420.0485148785, - "particle_momentum_x": 1.468116154656724e-17, - "particle_momentum_y": 1.4650318746367807e-17, - "particle_momentum_z": 1.1638654342620123e-17, - "particle_weight": 1175692137613.312 + "particle_position_x": 751412.1033107714, + "particle_position_y": 751687.9554423956, + "particle_position_z": 644516.6066265699, + "particle_momentum_x": 1.4698270572291615e-17, + "particle_momentum_y": 1.466699162579312e-17, + "particle_momentum_z": 1.162881798230096e-17, + "particle_weight": 1176011216106.0706 } } - - diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index 454edfc2606..727c8beb7f7 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -9,11 +9,11 @@ import json import sys -from benchmark import Benchmark import numpy as np +import yt +from benchmark import Benchmark from openpmd_viewer import OpenPMDTimeSeries from scipy.constants import c -import yt yt.funcs.mylog.setLevel(50) diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 2f0c7b607bb..855dcff58fd 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.03 +branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index d6ac81c5607..ac321b8b694 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.03 +branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -465,6 +465,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +[ElectrostaticSphereLabFrame_MR_emass_10] +buildDir = . +inputFile = Examples/Tests/electrostatic_sphere/inputs_3d +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 +dim = 3 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=3 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py + [ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_3d @@ -554,7 +571,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w phi dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -605,7 +622,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sph [embedded_boundary_cube] buildDir = . inputFile = Examples/Tests/embedded_boundary_cube/inputs_3d -runtime_params = +runtime_params = warpx.abort_on_warning_threshold = medium dim = 3 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON @@ -622,7 +639,7 @@ analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_cube_2d] buildDir = . inputFile = Examples/Tests/embedded_boundary_cube/inputs_2d -runtime_params = +runtime_params = warpx.abort_on_warning_threshold = medium dim = 2 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON @@ -639,7 +656,7 @@ analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py [embedded_boundary_cube_macroscopic] buildDir = . inputFile = Examples/Tests/embedded_boundary_cube/inputs_3d -runtime_params = algo.em_solver_medium=macroscopic macroscopic.epsilon=1.5*8.8541878128e-12 macroscopic.sigma=0 macroscopic.mu=1.25663706212e-06 +runtime_params = algo.em_solver_medium=macroscopic macroscopic.epsilon=1.5*8.8541878128e-12 macroscopic.sigma=0 macroscopic.mu=1.25663706212e-06 warpx.abort_on_warning_threshold=medium dim = 3 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON @@ -1189,7 +1206,7 @@ analysisOutputImage = langmuir_fluid_multi_analysis.png [Langmuir_multi_1d] buildDir = . inputFile = Examples/Tests/langmuir/inputs_1d -runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=z w ux uy uz diag1.positrons.variables=z w ux uy uz dim = 1 addToCompileString = USE_OPENPMD=TRUE QED=FALSE cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF @@ -1208,7 +1225,7 @@ analysisOutputImage = langmuir_multi_1d_analysis.png [Langmuir_multi_2d_MR] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz +runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1227,7 +1244,7 @@ analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_anisotropic] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz +runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1246,7 +1263,7 @@ analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1265,7 +1282,7 @@ analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png [Langmuir_multi_2d_MR_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1284,7 +1301,7 @@ analysisOutputImage = Langmuir_multi_2d_MR_psatd.png [Langmuir_multi_2d_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1303,7 +1320,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1322,7 +1339,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1341,7 +1358,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1360,7 +1377,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1417,7 +1434,7 @@ analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png [Langmuir_multi_2d_psatd_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1436,7 +1453,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1455,7 +1472,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1474,7 +1491,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1492,7 +1509,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d +inputFile = runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = @@ -1721,7 +1738,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_rz] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 +runtime_params = diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ @@ -1741,7 +1758,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1761,7 +1778,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1781,7 +1798,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_multiJ] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 +runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -4189,6 +4206,40 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py +[resample_velocity_coincidence_thinning] +buildDir = . +inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/analysis_default_regression.py + +[resample_velocity_coincidence_thinning_cartesian] +buildDir = . +inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/analysis_default_regression.py + [restart] buildDir = . inputFile = Examples/Tests/restart/inputs @@ -4720,3 +4771,38 @@ compareParticles = 1 particleTypes = electrons outputFile = particle_boundary_interaction_plt analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py + +[particle_thermal_boundary] +buildDir = . +inputFile = Examples/Tests/particle_thermal_boundary/inputs_2d +runtime_params = +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py + +[openbc_poisson_solver] +buildDir = . +inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +doComparison = 0 +analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index d52d05b1139..ce7a3398b95 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -6,6 +6,7 @@ # License: BSD-3-Clause-LBNL import os + # This script modifies `WarpX-test.ini` (which is used for nightly builds) # and creates the file `ci-test.ini` (which is used for continuous # integration) diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 5f41ecbd706..42a96f3628a 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -11,6 +11,8 @@ #include "PML_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif @@ -131,13 +133,11 @@ private: amrex::Real dt_E = -1.e10; }; -enum struct PatchType : int; - class PML { public: - PML (int lev, - const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + PML (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, bool do_similar_dm_pml, const amrex::Geometry* geom, const amrex::Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 805b4fec181..25b34818dd1 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -10,6 +10,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" +#include "FieldSolver/Fields.H" #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif @@ -55,6 +56,7 @@ #endif using namespace amrex; +using namespace warpx::fields; namespace { @@ -542,7 +544,8 @@ MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) } } -PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& grid_dm, +PML::PML (const int lev, const BoxArray& grid_ba, + const DistributionMapping& grid_dm, const bool do_similar_dm_pml, const Geometry* geom, const Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, @@ -658,7 +661,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } DistributionMapping dm; - if (WarpX::do_similar_dm_pml) { + if (do_similar_dm_pml) { auto ng_sim = amrex::elemwiseMax(amrex::elemwiseMax(nge, ngb), ngf); dm = amrex::MakeSimilarDM(ba, grid_ba, grid_dm, ng_sim); } else { @@ -678,23 +681,23 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const int ncompe = (m_dive_cleaning) ? 3 : 2; const int ncompb = (m_divb_cleaning) ? 3 : 2; - const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); - const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); @@ -806,22 +809,23 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const BoxArray& cba = MakeBoxArray(is_single_box_domain, cdomain, *cgeom, grid_cba_reduced, cncells, do_pml_in_domain, do_pml_Lo, do_pml_Hi); DistributionMapping cdm; - if (WarpX::do_similar_dm_pml) { + if (do_similar_dm_pml) { auto ng_sim = amrex::elemwiseMax(amrex::elemwiseMax(nge, ngb), ngf); cdm = amrex::MakeSimilarDM(cba, grid_cba_reduced, grid_dm, ng_sim); } else { cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,2).ixType().toIntVect()); + + const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_cp[0], cba_Ex, cdm, ncompe, nge, lev, "pml_E_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[1], cba_Ey, cdm, ncompe, nge, lev, "pml_E_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[2], cba_Ez, cdm, ncompe, nge, lev, "pml_E_cp[z]", 0.0_rt); - const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,2).ixType().toIntVect()); + const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_cp[0], cba_Bx, cdm, ncompb, ngb, lev, "pml_B_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[1], cba_By, cdm, ncompb, ngb, lev, "pml_B_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[2], cba_Bz, cdm, ncompb, ngb, lev, "pml_B_cp[z]", 0.0_rt); @@ -842,9 +846,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,2).ixType().toIntVect()); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 3a5a51770f8..74d3da2e19b 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -10,6 +10,8 @@ #include "PML_RZ_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif @@ -25,8 +27,6 @@ #include #include -enum struct PatchType : int; - class PML_RZ { public: diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 7eb011a6bb2..faa7609b3ab 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -8,6 +8,7 @@ #include "PML_RZ.H" #include "BoundaryConditions/PML_RZ.H" +#include "FieldSolver/Fields.H" #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif @@ -33,6 +34,7 @@ #include using namespace amrex; +using namespace warpx::fields; PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, const amrex::Geometry* geom, const int ncell, const int do_pml_in_domain) @@ -41,15 +43,15 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist m_geom(geom) { - const amrex::MultiFab & Er_fp = WarpX::GetInstance().getEfield_fp(lev,0); - const amrex::MultiFab & Et_fp = WarpX::GetInstance().getEfield_fp(lev,1); + const amrex::MultiFab & Er_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,0); + const amrex::MultiFab & Et_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,1); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); - const amrex::MultiFab & Br_fp = WarpX::GetInstance().getBfield_fp(lev,0); - const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getBfield_fp(lev,1); + const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); + const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); diff --git a/Source/BoundaryConditions/PML_current.H b/Source/BoundaryConditions/PML_current.H index 88e94de1e74..9361ce13992 100644 --- a/Source/BoundaryConditions/PML_current.H +++ b/Source/BoundaryConditions/PML_current.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PML_CURRENT_H_ -#define PML_CURRENT_H_ +#ifndef WARPX_PML_CURRENT_H_ +#define WARPX_PML_CURRENT_H_ #include "BoundaryConditions/PMLComponent.H" @@ -143,4 +143,4 @@ void damp_jz_pml (int j, int k, int l, #endif } -#endif +#endif //WARPX_PML_CURRENT_H_ diff --git a/Source/BoundaryConditions/PML_fwd.H b/Source/BoundaryConditions/PML_fwd.H index 58532a89070..21ad81949f1 100644 --- a/Source/BoundaryConditions/PML_fwd.H +++ b/Source/BoundaryConditions/PML_fwd.H @@ -14,8 +14,6 @@ struct SigmaBox; class SigmaBoxFactory; class MultiSigmaBox; -enum struct PatchType; - class PML; #endif /* WARPX_PML_FWD_H */ diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index af7a385b729..a5e0004325a 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -15,57 +15,113 @@ #include #include #include -using namespace amrex::literals; + using namespace amrex; +using namespace amrex::literals; +using namespace warpx::fields; + +namespace +{ + /** Returns true if any field boundary is set to FieldBoundaryType FT, else returns false.*/ + template + [[nodiscard]] + bool isAnyBoundary (const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi) + { + const auto isFT = [](const auto& b){ + return b == FT;}; + return std::any_of(field_boundary_lo.begin(), field_boundary_lo.end(), isFT) || + std::any_of(field_boundary_hi.begin(), field_boundary_hi.end(), isFT); + } + + /** Returns true if any particle boundary is set to ParticleBoundaryType PT, else returns false.*/ + template + [[nodiscard]] + bool isAnyBoundary (const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi) + { + const auto isPT = [](const auto& b){ + return b == PT;}; + return std::any_of(particle_boundary_lo.begin(), particle_boundary_lo.end(), isPT) || + std::any_of(particle_boundary_hi.begin(), particle_boundary_hi.end(), isPT); + } + +} void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoEfield( { get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2) }, lev, patch_type); - if (WarpX::isAnyBoundaryPML()) { + PEC::ApplyPECtoEfield( + {getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( pml[lev]->GetE_fp(), lev, patch_type, split_pml_field); + PEC::ApplyPECtoEfield( + pml[lev]->GetE_fp(), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); } } else { - PEC::ApplyPECtoEfield( { get_pointer_Efield_cp(lev, 0), - get_pointer_Efield_cp(lev, 1), - get_pointer_Efield_cp(lev, 2) }, lev, patch_type); - if (WarpX::isAnyBoundaryPML()) { + PEC::ApplyPECtoEfield( + {getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( pml[lev]->GetE_cp(), lev, patch_type, split_pml_field); + PEC::ApplyPECtoEfield( + pml[lev]->GetE_cp(), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); } } } #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2), lev); } else { - ApplyFieldBoundaryOnAxis(get_pointer_Efield_cp(lev, 0), - get_pointer_Efield_cp(lev, 1), - get_pointer_Efield_cp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2), lev); } #endif } void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoBfield( { get_pointer_Bfield_fp(lev, 0), - get_pointer_Bfield_fp(lev, 1), - get_pointer_Bfield_fp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { + getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2) }, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); } else { - PEC::ApplyPECtoBfield( { get_pointer_Bfield_cp(lev, 0), - get_pointer_Bfield_cp(lev, 1), - get_pointer_Bfield_cp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { + getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); } } @@ -74,31 +130,24 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d // E and B are staggered in time, which is only true after the first half-push if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { - bool applySilverMueller = false; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( (WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller) || - (WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller) ) { - applySilverMueller = true; - } - } - if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp[lev], Bfield_fp[lev], - Geom(lev).Domain(), dt[lev], - WarpX::field_boundary_lo, - WarpX::field_boundary_hi); + if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ + m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + Efield_fp[lev], Bfield_fp[lev], + Geom(lev).Domain(), dt[lev], + field_boundary_lo, field_boundary_hi); } } } #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(get_pointer_Bfield_fp(lev, 0), - get_pointer_Bfield_fp(lev, 1), - get_pointer_Bfield_fp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2), lev); } else { - ApplyFieldBoundaryOnAxis(get_pointer_Bfield_cp(lev, 0), - get_pointer_Bfield_cp(lev, 1), - get_pointer_Bfield_cp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2), lev); } #endif } @@ -106,14 +155,30 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoRhofield(rho, lev, patch_type); } + if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) + { + PEC::ApplyReflectiveBoundarytoRhofield(rho, + field_boundary_lo, field_boundary_hi, + particle_boundary_lo, particle_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); + } } void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } + if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) + { + PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, + field_boundary_lo, field_boundary_hi, + particle_boundary_lo, particle_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); + } } #ifdef WARPX_DIM_RZ @@ -201,11 +266,12 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoElectronPressure( - m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), lev, patch_type - ); + m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), + field_boundary_lo, field_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } else { amrex::Abort(Utils::TextMsg::Err( "ApplyElectronPressureBoundary: Only one level implemented for hybrid solver.")); diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index 62b92ba94b5..a6af894beb4 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -1,16 +1,11 @@ #ifndef WARPX_PEC_KERNELS_H_ #define WARPX_PEC_KERNELS_H_ -#include "WarpX.H" #include "Utils/WarpXAlgorithmSelection.H" #include -#include -#include -#include -#include -#include -#include +#include +#include #include @@ -18,524 +13,113 @@ #include namespace PEC { -using namespace amrex; - /** - * \brief Determines if the field boundary condition stored in fboundary - * in direction, dir, is PEC. - * - * \param[in] fboundary Value containing boundary type - * \param[in] dir direction - * - * \returns 1 if the boundary type is PEC else 0 - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - bool is_boundary_PEC (amrex::GpuArray const& fboundary, int dir) { - return ( fboundary[dir] == FieldBoundaryType::PEC ); - } - - /** - * \brief Determines if the particle boundary condition stored in pboundary - * in direction, dir, is reflecting. - * - * \param[in] pboundary Value containing boundary type - * \param[in] dir direction - * - * \returns 1 if the boundary type is reflecting else 0 - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - bool is_boundary_reflecting (amrex::GpuArray const& pboundary, int dir) { - return ( pboundary[dir] == ParticleBoundaryType::Reflecting ); - } - - /** - * \brief Calculates the number of grid points the given index is pass the - * domain boundary i.e. a value of +1 means the current cell is - * outside of the simulation domain by 1 cell. Note that the high - * side domain boundary is between cell dom_hi and dom_hi+1 for cell - * centered grids and on cell dom_hi+1 for nodal grid. This is why - * (dom_hi[idim] + is_nodal[idim]) is used below. - * - * \param[in] dom_lo, dom_hi Domain boundaries - * \param[in] ijk_vec Cell coordinates - * \param[in] is_nodal Whether the field of interest is nodal - * \param[in] idim Dimension of interest - * \param[in] iside 0 for low and 1 for high - * - * \returns number of grid points to the boundary - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - int get_cell_count_to_boundary (const amrex::IntVect& dom_lo, - const amrex::IntVect& dom_hi, const amrex::IntVect& ijk_vec, - const amrex::IntVect& is_nodal, const int idim, const int iside) - { - return ((iside == 0) ? (dom_lo[idim] - ijk_vec[idim]) - : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); - } - - /** - * \brief Sets the electric field value tangential to the PEC boundary to zero. The - * tangential Efield components in the guard cells outside the - * domain boundary are set equal and opposite to the field in the valid cells - * at their mirrored locations. The normal Efield components in the guard cells - * are set equal to the field in the valid cells at their mirrored locations. - * The number or depth of guard cells updated is equal to the shape factor of - * particles in each dimension. - * For corner cells with mixed boundaries, the mirror location could be outside - * valid region, while still ensuring PEC condition is maintained across the - * PEC boundary, and the necessary sign change is accounted for depending on - * if the component, icomp, is tangential or normal to the PEC boundary. - * - * For 3D : - * x component is tangential to the y-boundary and z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary and y-boundary - * x component is normal to the x-boundary - * y component is normal to the y-boundary - * z component is normal to the z-boundary - * where, x-boundary is the yz-plane at x=xmin and x=xmax - * y-boundary is the xz-plane at y=ymin and y=ymax - * z-boundary is the xy-plane at z=zmin and z=zmax - * - * For 2D : WarpX uses X-Z as the two dimensions - * x component is tangential to the z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary - * x component is normal to the x-boundary - * y component is not normal to any boundary (Only xz dimensions in 2D) - * z component is normal to the z-boundary - * where, x-boundary is along the line z at x=xmin and x=xmax - * z-boundary is along the line x at z=zmin and z=zmax - * - * For 1D : WarpX uses Z as the only dimension - * x component is tangential to the z-boundary - * y component is tangential to the z-boundary - * z component is not tangential to the z-boundary - * x component is not normal to any boundary (Only z dimension in 1D) - * y component is not normal to any boundary (Only z dimension in 1D) - * z component is normal to the z-boundary - * where, z-boundary is a point at z=zmin and z=zmax - * - * For RZ : WarpX uses R-Z as the two dimensions - * r component is tangential to the z-boundary - * theta_component is tangential to the r-boundary and z-boundary - * z component is tangential to the r-boundary - * r component is normal to the r-boundary - * theta_component is not normal to any boundary (on RZ dimensions are modeled) - * z component is normal to the z-boundary - * where, r-boundary is along the line z at r=rmin and r=rmax - * z-boundary is along the line r at z=zmin and z=zmax - * - * - * \param[in] icomp component of the Efield being updated - * (0=x, 1=y, 2=z in Cartesian) - * (0=r, 1=theta, 2=z in RZ) - * \param[in] dom_lo index value of the lower domain boundary (cell-centered) - * \param[in] dom_hi index value of the higher domain boundary (cell-centered) - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 - * \param[in] n index of the MultiFab component being updated - * \param[in] Efield field data to be updated if (ijk) is at the boundary or a guard cell - * \param[in] is_nodal staggering of the field data being updated. - * \param[in] fbndry_lo Field boundary type at the lower boundaries - * \param[in] fbndry_hi Field boundary type at the upper boundaries - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetEfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, - const amrex::IntVect &dom_hi, - const amrex::IntVect &ijk_vec, const int n, - amrex::Array4 const& Efield, - const amrex::IntVect& is_nodal, - amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) - { - // Tangential Efield components in guard cells set equal and opposite to cells - // in the mirror locations across the PEC boundary, whereas normal E-field - // components are set equal to values in the mirror locations across the PEC - // boundary. Here we just initialize it. - amrex::IntVect ijk_mirror = ijk_vec; - bool OnPECBoundary = false; - bool GuardCell = false; - amrex::Real sign = 1._rt; - // Loop over all the dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Loop over sides, iside = 0 (lo), iside = 1 (hi) - for (int iside = 0; iside < 2; ++iside) { - const bool isPECBoundary = ( (iside == 0) - ? is_boundary_PEC(fbndry_lo, idim) - : is_boundary_PEC(fbndry_hi, idim) ); -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), - // icomp=1 is tangential to both x and z boundaries - // The logic below ensures that the flags are set right for 2D - const bool is_tangent_to_PEC = (icomp != AMREX_SPACEDIM*idim); -#elif (defined WARPX_DIM_1D_Z) - // For 1D : icomp=0 and icomp=1 (Ex and Ey are tangential to the z boundary) - // The logic below ensures that the flags are set right for 1D - const bool is_tangent_to_PEC = (icomp != idim+2); -#else - const bool is_tangent_to_PEC = (icomp != idim); -#endif - if (isPECBoundary) { - // grid point ijk_vec is ig number of points pass the - // domain boundary in direction, idim - const int ig = get_cell_count_to_boundary( - dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); - - if (ig == 0) { - if (is_tangent_to_PEC && is_nodal[idim] == 1) { - OnPECBoundary = true; - } - } else if (ig > 0) { - // Find mirror location across PEC boundary - ijk_mirror[idim] = ( ( iside == 0) - ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) - : (dom_hi[idim] + 1 - ig)); - GuardCell = true; - // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) { sign *= -1._rt; } -#if (defined WARPX_DIM_RZ) - if (icomp == 0 && idim == 0 && iside == 1) { - // Add radial scale so that drEr/dr = 0. - // This only works for the first guard cell and with - // Er cell centered in r. - const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - sign *= rmirror/rguard; - } -#endif - } - } // is PEC boundary - } // loop over iside - } // loop over dimensions - if (OnPECBoundary) { - // if ijk_vec is on a PEC boundary in any direction, set Etangential to 0. - Efield(ijk_vec,n) = 0._rt; - } else if (GuardCell) { - Efield(ijk_vec,n) = sign * Efield(ijk_mirror,n); - } - } - - - /** - * \brief Sets the magnetic field value normal to the PEC boundary to zero. The - * tangential (and normal) field value of the guard cells outside the - * domain boundary are set equal (and opposite) to the respective field components - * in the valid cells at their mirrored locations. - * The number or depth of guard cells updated is equal to the shape factor of - * particles in each dimension. - * - * For 3D : - * x component is tangential to the y-boundary and z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary and y-boundary - * x component is normal to the x-boundary - * y component is normal to the y-boundary - * z component is normal to the z-boundary - * where, x-boundary is the yz-plane at x=xmin and x=xmax - * y-boundary is the xz-plane at y=ymin and y=ymax - * z-boundary is the xy-plane at z=zmin and z=zmax - * - * For 2D : WarpX uses X-Z as the two dimensions - * x component is tangential to the z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary - * x component is normal to the x-boundary - * y component is not normal to any boundary (Only xz dimensions in 2D) - * z component is normal to the z-boundary - * where, x-boundary is along the line z at x=xmin and x=xmax - * z-boundary is along the line x at z=zmin and z=zmax - * - * For 1D : WarpX uses Z as the only dimension - * x component is tangential to the z-boundary - * y component is tangential to the z-boundary - * z component is not tangential to the z-boundary - * x component is not normal to any boundary (Only z dimension in 1D) - * y component is not normal to any boundary (Only z dimension in 1D) - * z component is normal to the z-boundary - * where, z-boundary is a point at z=zmin and z=zmax - * - * For RZ : WarpX uses R-Z as the two dimensions - * r component is tangential to the z-boundary - * theta_component is tangential to the r-boundary and z-boundary - * z component is tangential to the r-boundary - * r component is normal to the r-boundary - * theta_component is not normal to any boundary (on RZ dimensions are modeled) - * z component is normal to the z-boundary - * where, r-boundary is along the line z at r=rmin and r=rmax - * z-boundary is along the line r at z=zmin and z=zmax - * - * - * \param[in] icomp component of the Bfield being updated - * (0=x, 1=y, 2=z in Cartesian) - * (0=r, 1=theta, 2=z in RZ) - * \param[in] dom_lo index value of the lower domain boundary (cell-centered) - * \param[in] dom_hi index value of the higher domain boundary (cell-centered) - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 - * \param[in] n index of the MultiFab component being updated - * \param[in] Bfield field data to be updated if (ijk) is at the boundary - or a guard cell - * \param[in] is_nodal staggering of the field data being updated. - * \param[in] fbndry_lo Field boundary type at the lower boundaries - * \param[in] fbndry_hi Field boundary type at the upper boundaries - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetBfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, - const amrex::IntVect & dom_hi, - const amrex::IntVect & ijk_vec, const int n, - amrex::Array4 const& Bfield, - const amrex::IntVect & is_nodal, - amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) - { - amrex::IntVect ijk_mirror = ijk_vec; - bool OnPECBoundary = false; - bool GuardCell = false; - amrex::Real sign = 1._rt; - // Loop over all dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Loop over sides, iside = 0 (lo), iside = 1 (hi) - for (int iside = 0; iside < 2; ++iside) { - const bool isPECBoundary = ( (iside == 0 ) - ? is_boundary_PEC(fbndry_lo, idim) - : is_boundary_PEC(fbndry_hi, idim) ); - if (isPECBoundary) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - // For 2D : for icomp==1, (By in XZ, Btheta in RZ), - // icomp=1 is not normal to x or z boundary - // The logic below ensures that the flags are set right for 2D - const bool is_normal_to_PEC = (icomp == (AMREX_SPACEDIM*idim)); -#elif (defined WARPX_DIM_1D_Z) - // For 1D : icomp=0 and icomp=1 (Bx and By are not normal to the z boundary) - // The logic below ensures that the flags are set right for 1D - const bool is_normal_to_PEC = (icomp == (idim+2)); -#else - const bool is_normal_to_PEC = (icomp == idim); -#endif - - // grid point ijk_vec is ig number of points pass the - // domain boundary in direction, idim - const int ig = get_cell_count_to_boundary( - dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); - - if (ig == 0) { - // Only normal component is set to 0 - if (is_normal_to_PEC && is_nodal[idim]==1) { - OnPECBoundary = true; - } - } else if ( ig > 0) { - // Mirror location inside the domain by "ig" number of cells - // across PEC boundary in direction, idim, and side, iside - ijk_mirror[idim] = ( (iside == 0) - ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) - : (dom_hi[idim] + 1 - ig)); - GuardCell = true; - // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) { sign *= -1._rt; } -#if (defined WARPX_DIM_RZ) - if (icomp == 0 && idim == 0 && iside == 1) { - // Add radial scale so that drBr/dr = 0. - const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - sign *= rmirror/rguard; - } -#endif - } - } // if PEC Boundary - } // loop over sides - } // loop of dimensions - - if (OnPECBoundary) { - // if ijk_vec is on a PEC boundary in any direction, set Bnormal to 0. - Bfield(ijk_vec,n) = 0._rt; - } else if (GuardCell) { - // Bnormal and Btangential is set opposite and equal to the value - // in the mirror location, respectively. - Bfield(ijk_vec,n) = sign * Bfield(ijk_mirror,n); - } - } - - - /** - * \brief Sets the rho or J field value in cells close to and on a PEC boundary. The - * charge/current density deposited in the guard cells are either reflected - * back into the simulation domain (if a reflecting particle - * boundary is used), or the opposite charge/current density is deposited - * back in the domain to capture the effect of an image charge. - * The charge/current density on the PEC boundary is set to 0 while values - * in the guard cells are set equal (and opposite) to their mirror - * location inside the domain - representing image charges - in the - * normal (tangential) direction. - * - * \param[in] n index of the MultiFab component being updated - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 - * \param[in out] field field data to be updated - * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec - * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_pec Whether the given boundary is PEC - * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary - * \param[in] fabbox multifab box including ghost cells - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetRhoOrJfieldFromPEC (const int n, - const amrex::IntVect & ijk_vec, - amrex::Array4 const& field, - amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, - amrex::GpuArray, AMREX_SPACEDIM> const& psign, - amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, - amrex::GpuArray const& tangent_to_bndy, - amrex::Box const& fabbox) - { - // The boundary is handled in 2 steps: - // 1) The cells internal to the domain are updated using the - // current deposited in the guard cells - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_pec[idim][iside]) { continue; } - - // Get the mirror guard cell index - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - - // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) { - field(ijk_vec, n) = 0._rt; - // otherwise update the internal cell if the mirror guard cell exists - } else if (fabbox.contains(iv_mirror)) { - field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); - } - } - } - // 2) The guard cells are updated with the appropriate image - // charge based on the charge/current in the valid cells - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_pec[idim][iside]) { continue; } - - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) - { - if (tangent_to_bndy[idim]) { - field(iv_mirror, n) = -field(ijk_vec, n); - } else { - field(iv_mirror, n) = field(ijk_vec, n); - } - } - } - } - } - - - /** - * \brief This function sets the given field value on a PEC boundary - * to enforce a Neumann boundary condition (zero derivative) in the - * normal direction. - * - * \param[in] n index of the MultiFab component being updated - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 - * \param[in out] field field data to be updated - * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec - * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_pec Whether the given boundary is PEC - * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary - * \param[in] fabbox multifab box including ghost cells - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetNeumannOnPEC (const int n, - const amrex::IntVect & ijk_vec, - amrex::Array4 const& field, - amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, - amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, - amrex::Box const& fabbox ) - { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_pec[idim][iside]) { continue; } - - // Get the mirror guard cell index - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - - // On the PEC boundary the field value is set equal to the - // first value in the domain (nodal fields) - if (ijk_vec == iv_mirror) { - iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } - } - // otherwise set the mirror guard cell equal to the internal cell value - else if (fabbox.contains(iv_mirror)) - { - field(iv_mirror, n) = field(ijk_vec, n); - } - } - } - } - - - /** Returns 1 if any domain boundary is set to PEC, else returns 0.*/ - bool isAnyBoundaryPEC(); /** * \brief Sets the tangential electric field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell * field value at the respective mirror locations. * - * \param[in,out] Efield Boundary values of tangential Efield are set to zero - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine - * \param[in] split_pml_field whether pml the multifab is the regular Efield or - * split pml field + * \param[in,out] Efield Boundary values of tangential Efield are set to zero + * \param[in] field_boundary_lo Boundary types of the "low" boundaries + * \param[in] field_boundary_hi Boundary types of the "high" boundaries + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field */ - void ApplyPECtoEfield ( std::array Efield, - int lev, PatchType patch_type, - bool split_pml_field = false); + void ApplyPECtoEfield ( + std::array Efield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + bool split_pml_field = false); /** * \brief Sets the normal component of the magnetic field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell * field value at the respective mirror locations. * - * \param[in,out] Bfield Boundary values of normal Bfield are set to zero. - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Bfield Boundary values of normal Bfield are set to zero. + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoBfield ( std::array Bfield, - int lev, PatchType patch_type); + void ApplyPECtoBfield ( + std::array Bfield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Reflects charge density deposited over the PEC boundary back into * the simulation domain. * - * \param[in,out] rho Multifab containing the charge density - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] rho Multifab containing the charge density + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] particle_boundary_lo Boundary types of the "low" particle boundaries + * \param[in] particle_boundary_hi Boundary types of the "high" particle boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoRhofield(amrex::MultiFab* rho, int lev, - PatchType patch_type); + void ApplyReflectiveBoundarytoRhofield( + amrex::MultiFab* rho, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Reflects current density deposited over the PEC boundary back into * the simulation domain. * - * \param[in,out] Jx, Jy, Jz Multifabs containing the current density - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Jx, Jy, Jz Multifabs containing the current density + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] particle_boundary_lo Boundary types of the "low" particle boundaries + * \param[in] particle_boundary_hi Boundary types of the "high" particle boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, - amrex::MultiFab* Jz, int lev, - PatchType patch_type); + void ApplyReflectiveBoundarytoJfield( + amrex::MultiFab* Jx, amrex::MultiFab* Jy, + amrex::MultiFab* Jz, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Apply the PEC boundary to the electron pressure field. * - * \param[in,out] Pefield Multifab containing the electron pressure - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Pefield Multifab containing the electron pressure + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, - int lev, PatchType patch_type); + void ApplyPECtoElectronPressure ( + amrex::MultiFab* Pefield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); } #endif // WarpX_PEC_KERNELS_H_ diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 7555e2aaef2..0067f54d3ff 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -1,51 +1,483 @@ #include "BoundaryConditions/WarpX_PEC.H" -#include "WarpX.H" #include +#include #include +#include +#include #include #include #include +#include #include +#include #include #include +#include #include -#include +using namespace amrex; using namespace amrex::literals; -bool -PEC::isAnyBoundaryPEC() { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } +namespace +{ + /** + * \brief Calculates the number of grid points the given index is pass the + * domain boundary i.e. a value of +1 means the current cell is + * outside of the simulation domain by 1 cell. Note that the high + * side domain boundary is between cell dom_hi and dom_hi+1 for cell + * centered grids and on cell dom_hi+1 for nodal grid. This is why + * (dom_hi[idim] + is_nodal[idim]) is used below. + * + * \param[in] dom_lo, dom_hi Domain boundaries + * \param[in] ijk_vec Cell coordinates + * \param[in] is_nodal Whether the field of interest is nodal + * \param[in] idim Dimension of interest + * \param[in] iside 0 for low and 1 for high + * + * \returns number of grid points to the boundary + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + int get_cell_count_to_boundary (const amrex::IntVect& dom_lo, + const amrex::IntVect& dom_hi, const amrex::IntVect& ijk_vec, + const amrex::IntVect& is_nodal, const int idim, const int iside) + { + return ((iside == 0) ? (dom_lo[idim] - ijk_vec[idim]) + : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); + } + + + /** + * \brief Sets the electric field value tangential to the PEC boundary to zero. The + * tangential Efield components in the guard cells outside the + * domain boundary are set equal and opposite to the field in the valid cells + * at their mirrored locations. The normal Efield components in the guard cells + * are set equal to the field in the valid cells at their mirrored locations. + * The number or depth of guard cells updated is equal to the shape factor of + * particles in each dimension. + * For corner cells with mixed boundaries, the mirror location could be outside + * valid region, while still ensuring PEC condition is maintained across the + * PEC boundary, and the necessary sign change is accounted for depending on + * if the component, icomp, is tangential or normal to the PEC boundary. + * + * For 3D : + * x component is tangential to the y-boundary and z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary and y-boundary + * x component is normal to the x-boundary + * y component is normal to the y-boundary + * z component is normal to the z-boundary + * where, x-boundary is the yz-plane at x=xmin and x=xmax + * y-boundary is the xz-plane at y=ymin and y=ymax + * z-boundary is the xy-plane at z=zmin and z=zmax + * + * For 2D : WarpX uses X-Z as the two dimensions + * x component is tangential to the z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary + * x component is normal to the x-boundary + * y component is not normal to any boundary (Only xz dimensions in 2D) + * z component is normal to the z-boundary + * where, x-boundary is along the line z at x=xmin and x=xmax + * z-boundary is along the line x at z=zmin and z=zmax + * + * For 1D : WarpX uses Z as the only dimension + * x component is tangential to the z-boundary + * y component is tangential to the z-boundary + * z component is not tangential to the z-boundary + * x component is not normal to any boundary (Only z dimension in 1D) + * y component is not normal to any boundary (Only z dimension in 1D) + * z component is normal to the z-boundary + * where, z-boundary is a point at z=zmin and z=zmax + * + * For RZ : WarpX uses R-Z as the two dimensions + * r component is tangential to the z-boundary + * theta_component is tangential to the r-boundary and z-boundary + * z component is tangential to the r-boundary + * r component is normal to the r-boundary + * theta_component is not normal to any boundary (on RZ dimensions are modeled) + * z component is normal to the z-boundary + * where, r-boundary is along the line z at r=rmin and r=rmax + * z-boundary is along the line r at z=zmin and z=zmax + * + * + * \param[in] icomp component of the Efield being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] Efield field data to be updated if (ijk) is at the boundary or a guard cell + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] fbndry_lo Field boundary type at the lower boundaries + * \param[in] fbndry_hi Field boundary type at the upper boundaries + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetEfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, + const amrex::IntVect &dom_hi, + const amrex::IntVect &ijk_vec, const int n, + amrex::Array4 const& Efield, + const amrex::IntVect& is_nodal, + amrex::GpuArray const& fbndry_lo, + amrex::GpuArray const& fbndry_hi ) + { + // Tangential Efield components in guard cells set equal and opposite to cells + // in the mirror locations across the PEC boundary, whereas normal E-field + // components are set equal to values in the mirror locations across the PEC + // boundary. Here we just initialize it. + amrex::IntVect ijk_mirror = ijk_vec; + bool OnPECBoundary = false; + bool GuardCell = false; + amrex::Real sign = 1._rt; + // Loop over all the dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = 0 (lo), iside = 1 (hi) + for (int iside = 0; iside < 2; ++iside) { + const bool isPECBoundary = ( (iside == 0) + ? fbndry_lo[idim] == FieldBoundaryType::PEC + : fbndry_hi[idim] == FieldBoundaryType::PEC ); +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), + // icomp=1 is tangential to both x and z boundaries + // The logic below ensures that the flags are set right for 2D + const bool is_tangent_to_PEC = (icomp != AMREX_SPACEDIM*idim); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Ex and Ey are tangential to the z boundary) + // The logic below ensures that the flags are set right for 1D + const bool is_tangent_to_PEC = (icomp != idim+2); +#else + const bool is_tangent_to_PEC = (icomp != idim); +#endif + if (isPECBoundary) { + // grid point ijk_vec is ig number of points pass the + // domain boundary in direction, idim + const int ig = ::get_cell_count_to_boundary( + dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); + + if (ig == 0) { + if (is_tangent_to_PEC && is_nodal[idim] == 1) { + OnPECBoundary = true; + } + } else if (ig > 0) { + // Find mirror location across PEC boundary + ijk_mirror[idim] = ( ( iside == 0) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + GuardCell = true; + // tangential components are inverted across PEC boundary + if (is_tangent_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drEr/dr = 0. + // This only works for the first guard cell and with + // Er cell centered in r. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif + } + } // is PEC boundary + } // loop over iside + } // loop over dimensions + if (OnPECBoundary) { + // if ijk_vec is on a PEC boundary in any direction, set Etangential to 0. + Efield(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + Efield(ijk_vec,n) = sign * Efield(ijk_mirror,n); + } + } + + + /** + * \brief Sets the magnetic field value normal to the PEC boundary to zero. The + * tangential (and normal) field value of the guard cells outside the + * domain boundary are set equal (and opposite) to the respective field components + * in the valid cells at their mirrored locations. + * The number or depth of guard cells updated is equal to the shape factor of + * particles in each dimension. + * + * For 3D : + * x component is tangential to the y-boundary and z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary and y-boundary + * x component is normal to the x-boundary + * y component is normal to the y-boundary + * z component is normal to the z-boundary + * where, x-boundary is the yz-plane at x=xmin and x=xmax + * y-boundary is the xz-plane at y=ymin and y=ymax + * z-boundary is the xy-plane at z=zmin and z=zmax + * + * For 2D : WarpX uses X-Z as the two dimensions + * x component is tangential to the z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary + * x component is normal to the x-boundary + * y component is not normal to any boundary (Only xz dimensions in 2D) + * z component is normal to the z-boundary + * where, x-boundary is along the line z at x=xmin and x=xmax + * z-boundary is along the line x at z=zmin and z=zmax + * + * For 1D : WarpX uses Z as the only dimension + * x component is tangential to the z-boundary + * y component is tangential to the z-boundary + * z component is not tangential to the z-boundary + * x component is not normal to any boundary (Only z dimension in 1D) + * y component is not normal to any boundary (Only z dimension in 1D) + * z component is normal to the z-boundary + * where, z-boundary is a point at z=zmin and z=zmax + * + * For RZ : WarpX uses R-Z as the two dimensions + * r component is tangential to the z-boundary + * theta_component is tangential to the r-boundary and z-boundary + * z component is tangential to the r-boundary + * r component is normal to the r-boundary + * theta_component is not normal to any boundary (on RZ dimensions are modeled) + * z component is normal to the z-boundary + * where, r-boundary is along the line z at r=rmin and r=rmax + * z-boundary is along the line r at z=zmin and z=zmax + * + * + * \param[in] icomp component of the Bfield being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] Bfield field data to be updated if (ijk) is at the boundary + or a guard cell + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] fbndry_lo Field boundary type at the lower boundaries + * \param[in] fbndry_hi Field boundary type at the upper boundaries + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetBfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, + const amrex::IntVect & dom_hi, + const amrex::IntVect & ijk_vec, const int n, + amrex::Array4 const& Bfield, + const amrex::IntVect & is_nodal, + amrex::GpuArray const& fbndry_lo, + amrex::GpuArray const& fbndry_hi ) + { + amrex::IntVect ijk_mirror = ijk_vec; + bool OnPECBoundary = false; + bool GuardCell = false; + amrex::Real sign = 1._rt; + // Loop over all dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = 0 (lo), iside = 1 (hi) + for (int iside = 0; iside < 2; ++iside) { + const bool isPECBoundary = ( (iside == 0) + ? fbndry_lo[idim] == FieldBoundaryType::PEC + : fbndry_hi[idim] == FieldBoundaryType::PEC ); + if (isPECBoundary) { +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (By in XZ, Btheta in RZ), + // icomp=1 is not normal to x or z boundary + // The logic below ensures that the flags are set right for 2D + const bool is_normal_to_PEC = (icomp == (AMREX_SPACEDIM*idim)); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Bx and By are not normal to the z boundary) + // The logic below ensures that the flags are set right for 1D + const bool is_normal_to_PEC = (icomp == (idim+2)); +#else + const bool is_normal_to_PEC = (icomp == idim); +#endif + + // grid point ijk_vec is ig number of points pass the + // domain boundary in direction, idim + const int ig = ::get_cell_count_to_boundary( + dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); + + if (ig == 0) { + // Only normal component is set to 0 + if (is_normal_to_PEC && is_nodal[idim]==1) { + OnPECBoundary = true; + } + } else if ( ig > 0) { + // Mirror location inside the domain by "ig" number of cells + // across PEC boundary in direction, idim, and side, iside + ijk_mirror[idim] = ( (iside == 0) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + GuardCell = true; + // Sign of the normal component in guard cell is inverted + if (is_normal_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drBr/dr = 0. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif + } + } // if PEC Boundary + } // loop over sides + } // loop of dimensions + + if (OnPECBoundary) { + // if ijk_vec is on a PEC boundary in any direction, set Bnormal to 0. + Bfield(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + // Bnormal and Btangential is set opposite and equal to the value + // in the mirror location, respectively. + Bfield(ijk_vec,n) = sign * Bfield(ijk_mirror,n); + } + } + + + /** + * \brief Sets the rho or J field value in cells close to and on reflecting particle boundary + * or PEC field boundary. The charge/current density deposited + * in the guard cells are either reflected + * back into the simulation domain (if a reflecting particle + * boundary is used), or the opposite charge/current density is deposited + * back in the domain to capture the effect of an image charge. + * The charge/current density on the reflecting boundary is set to 0 while values + * in the guard cells are set equal (and opposite) to their mirror + * location inside the domain - representing image charges - in the + * normal (tangential) direction. + * + * \param[in] n index of the MultiFab component being updated + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 + * \param[in out] field field data to be updated + * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec + * \param[in] psign Whether the field value should be flipped across the boundary + * \param[in] is_reflective Whether the given particle boundary is reflecting or field boundary is pec + * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary + * \param[in] fabbox multifab box including ghost cells + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetRhoOrJfieldFromPEC (const int n, + const amrex::IntVect & ijk_vec, + amrex::Array4 const& field, + amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, + amrex::GpuArray, AMREX_SPACEDIM> const& psign, + amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, + amrex::GpuArray const& tangent_to_bndy, + amrex::Box const& fabbox) + { + // The boundary is handled in 2 steps: + // 1) The cells internal to the domain are updated using the + // current deposited in the guard cells + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_reflective[idim][iside]) { continue; } + + // Get the mirror guard cell index + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + + // On the PEC boundary the charge/current density is set to 0 + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; + // otherwise update the internal cell if the mirror guard cell exists + } else if (fabbox.contains(iv_mirror)) { + field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); + } + } + } + // 2) The guard cells are updated with the appropriate image + // charge based on the charge/current in the valid cells + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_reflective[idim][iside]) { continue; } + + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) + { + if (tangent_to_bndy[idim]) { + field(iv_mirror, n) = -field(ijk_vec, n); + } else { + field(iv_mirror, n) = field(ijk_vec, n); + } + } + } + } + } + + + /** + * \brief This function sets the given field value on a PEC boundary + * to enforce a Neumann boundary condition (zero derivative) in the + * normal direction. + * + * \param[in] n index of the MultiFab component being updated + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 + * \param[in out] field field data to be updated + * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec + * \param[in] psign Whether the field value should be flipped across the boundary + * \param[in] is_pec Whether the given boundary is PEC + * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary + * \param[in] fabbox multifab box including ghost cells + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetNeumannOnPEC (const int n, + const amrex::IntVect & ijk_vec, + amrex::Array4 const& field, + amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, + amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, + amrex::Box const& fabbox ) + { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_pec[idim][iside]) { continue; } + + // Get the mirror guard cell index + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + + // On the PEC boundary the field value is set equal to the + // first value in the domain (nodal fields) + if (ijk_vec == iv_mirror) { + iv_mirror[idim] += (iside == 0) ? 1 : -1; + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } + } + // otherwise set the mirror guard cell equal to the internal cell value + else if (fabbox.contains(iv_mirror)) + { + field(iv_mirror, n) = field(ijk_vec, n); + } + } + } } - return false; } void -PEC::ApplyPECtoEfield (std::array Efield, const int lev, - PatchType patch_type, const bool split_pml_field) +PEC::ApplyPECtoEfield ( + std::array Efield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + const bool split_pml_field) { - auto& warpx = WarpX::GetInstance(); - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } const amrex::IntVect domain_lo = domain_box.smallEnd(); const amrex::IntVect domain_hi = domain_box.bigEnd(); - amrex::GpuArray fbndry_lo; - amrex::GpuArray fbndry_hi; + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - fbndry_lo[idim] = WarpX::field_boundary_lo[idim]; - fbndry_hi[idim] = WarpX::field_boundary_hi[idim]; + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; } const amrex::IntVect Ex_nodal = Efield[0]->ixType().toIntVect(); const amrex::IntVect Ey_nodal = Efield[1]->ixType().toIntVect(); const amrex::IntVect Ez_nodal = Efield[2]->ixType().toIntVect(); - const amrex::IntVect ng_fieldgather = warpx.get_ng_fieldgather(); // For each Efield multifab, apply PEC boundary condition to ncomponents // If not split E-field, the PEC is applied to the regular Efield used in Maxwell's eq. // If split_pml_field is true, then PEC is applied to all the split field components of the tangential field. @@ -86,7 +518,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ex, Ex_nodal, fbndry_lo, fbndry_hi); }, tey, nComp_y, @@ -99,7 +531,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ey, Ey_nodal, fbndry_lo, fbndry_hi); }, tez, nComp_z, @@ -112,7 +544,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ez, Ez_nodal, fbndry_lo, fbndry_hi); } ); @@ -121,27 +553,28 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, void -PEC::ApplyPECtoBfield (std::array Bfield, const int lev, - PatchType patch_type) +PEC::ApplyPECtoBfield ( + std::array Bfield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } const amrex::IntVect domain_lo = domain_box.smallEnd(); const amrex::IntVect domain_hi = domain_box.bigEnd(); - amrex::GpuArray fbndry_lo; - amrex::GpuArray fbndry_hi; + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - fbndry_lo[idim] = WarpX::field_boundary_lo[idim]; - fbndry_hi[idim] = WarpX::field_boundary_hi[idim]; + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; } const amrex::IntVect Bx_nodal = Bfield[0]->ixType().toIntVect(); const amrex::IntVect By_nodal = Bfield[1]->ixType().toIntVect(); const amrex::IntVect Bz_nodal = Bfield[2]->ixType().toIntVect(); - const amrex::IntVect ng_fieldgather = warpx.get_ng_fieldgather(); const int nComp_x = Bfield[0]->nComp(); const int nComp_y = Bfield[1]->nComp(); const int nComp_z = Bfield[2]->nComp(); @@ -177,7 +610,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Bx, Bx_nodal, fbndry_lo, fbndry_hi); }, tby, nComp_y, @@ -190,7 +623,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, By, By_nodal, fbndry_lo, fbndry_hi); }, tbz, nComp_z, @@ -203,7 +636,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Bz, Bz_nodal, fbndry_lo, fbndry_hi); } ); @@ -222,14 +655,18 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, * location inside the domain - representing image charges. **/ void -PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_type) +PEC::ApplyReflectiveBoundarytoRhofield ( + amrex::MultiFab* rho, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } domain_box.convert(rho->ixType()); @@ -243,23 +680,29 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // cells for boundaries that are NOT PEC amrex::Box grown_domain_box = domain_box; - amrex::GpuArray, AMREX_SPACEDIM> is_pec; + amrex::GpuArray, AMREX_SPACEDIM> is_reflective; amrex::GpuArray is_tangent_to_bndy; amrex::GpuArray, AMREX_SPACEDIM> psign; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } - if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } + is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); + if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } // rho values inside guard cells are updated the same as tangential // components of the current density is_tangent_to_bndy[idim] = true; - psign[idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[idim][0] = ((particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; - psign[idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[idim][1] = ((particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - rho_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - rho_nodal[idim]); @@ -292,8 +735,8 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( - n, iv, rho_array, mirrorfac, psign, is_pec, + ::SetRhoOrJfieldFromPEC( + n, iv, rho_array, mirrorfac, psign, is_reflective, is_tangent_to_bndy, fabbox ); }); @@ -302,16 +745,18 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty void -PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, - amrex::MultiFab* Jz, const int lev, - PatchType patch_type) +PEC::ApplyReflectiveBoundarytoJfield( + amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } // Note: force domain box to be nodal to simplify the mirror cell @@ -335,15 +780,19 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // directions of the current density multifab const amrex::IntVect ng_fieldgather = Jx->nGrowVect(); - amrex::GpuArray, AMREX_SPACEDIM> is_pec; + amrex::GpuArray, AMREX_SPACEDIM> is_reflective; amrex::GpuArray, 3> is_tangent_to_bndy; amrex::GpuArray, AMREX_SPACEDIM>, 3> psign; amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } - if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } + is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); + if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } for (int icomp=0; icomp < 3; ++icomp) { // Set the psign value correctly for each current component for each @@ -362,15 +811,19 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, #endif if (is_tangent_to_bndy[icomp][idim]){ - psign[icomp][idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; - psign[icomp][idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; } else { - psign[icomp][idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? -1._rt : 1._rt; - psign[icomp][idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? -1._rt : 1._rt; } } @@ -387,6 +840,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, } // Each current component is handled separately below, starting with Jx. + grown_domain_box.convert(Jx_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -397,7 +851,6 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jx_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -414,14 +867,15 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( - n, iv, Jx_array, mirrorfac[0], psign[0], is_pec, + ::SetRhoOrJfieldFromPEC( + n, iv, Jx_array, mirrorfac[0], psign[0], is_reflective, is_tangent_to_bndy[0], fabbox ); }); } // Handle Jy. + grown_domain_box.convert(Jy_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -432,7 +886,6 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jy_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -449,14 +902,15 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( - n, iv, Jy_array, mirrorfac[1], psign[1], is_pec, + ::SetRhoOrJfieldFromPEC( + n, iv, Jy_array, mirrorfac[1], psign[1], is_reflective, is_tangent_to_bndy[1], fabbox ); }); } // Handle Jz. + grown_domain_box.convert(Jz_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -467,7 +921,6 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jz_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -484,8 +937,8 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( - n, iv, Jz_array, mirrorfac[2], psign[2], is_pec, + ::SetRhoOrJfieldFromPEC( + n, iv, Jz_array, mirrorfac[2], psign[2], is_reflective, is_tangent_to_bndy[2], fabbox ); }); @@ -493,15 +946,16 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, } void -PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, - PatchType patch_type) +PEC::ApplyPECtoElectronPressure ( + amrex::MultiFab* Pefield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } domain_box.convert(Pefield->ixType()); @@ -518,8 +972,8 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, amrex::GpuArray, AMREX_SPACEDIM> is_pec; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; + is_pec[idim][0] = field_boundary_lo[idim] == FieldBoundaryType::PEC; + is_pec[idim][1] = field_boundary_hi[idim] == FieldBoundaryType::PEC; if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -554,7 +1008,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetNeumannOnPEC(n, iv, Pe_array, mirrorfac, is_pec, fabbox); + ::SetNeumannOnPEC(n, iv, Pe_array, mirrorfac, is_pec, fabbox); }); } } diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 0a22774bb8d..7a1f8275bca 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef BTD_PLOTFILE_HEADER_IMPL_H -#define BTD_PLOTFILE_HEADER_IMPL_H +#ifndef WARPX_BTD_PLOTFILE_HEADER_IMPL_H +#define WARPX_BTD_PLOTFILE_HEADER_IMPL_H #include #include @@ -200,12 +200,12 @@ class BTDMultiFabHeaderImpl */ void SetBox (int ibox, amrex::Box ba_box) { m_ba.set(ibox, ba_box); } /** Set Fab name of the ith fab to be written in the multifab Header file.*/ - void SetFabName (int ifab, std::string fodPrefix, std::string FabName, + void SetFabName (int ifab, const std::string& fodPrefix, const std::string& FabName, int FabHead); /** Set minimum value of all the components for the ith fab. */ - void SetMinVal (int ifab, amrex::Vector minval); + void SetMinVal (int ifab, const amrex::Vector& minval); /** Set maximum value of all the components for the ith fab. */ - void SetMaxVal (int ifab, amrex::Vector maxval); + void SetMaxVal (int ifab, const amrex::Vector& maxval); private: /** Header file path */ std::string m_Header_path; @@ -342,4 +342,4 @@ public: std::string m_Header_path; }; -#endif +#endif //WARPX_BTD_PLOTFILE_HEADER_IMPL_H diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index cd3eb962405..4ec1c974f8e 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -323,7 +323,7 @@ BTDMultiFabHeaderImpl::ResizeFabData () } void -BTDMultiFabHeaderImpl::SetFabName (int ifab, std::string fodPrefix, std::string FabName, +BTDMultiFabHeaderImpl::SetFabName (int ifab, const std::string& fodPrefix, const std::string& FabName, int FabHead) { m_FabOnDiskPrefix[ifab] = fodPrefix; @@ -333,13 +333,13 @@ BTDMultiFabHeaderImpl::SetFabName (int ifab, std::string fodPrefix, std::string } void -BTDMultiFabHeaderImpl::SetMinVal (int ifab, amrex::Vector minval) +BTDMultiFabHeaderImpl::SetMinVal (int ifab, const amrex::Vector& minval) { CopyVec(m_minval[ifab], minval); } void -BTDMultiFabHeaderImpl::SetMaxVal (int ifab, amrex::Vector maxval) +BTDMultiFabHeaderImpl::SetMaxVal (int ifab, const amrex::Vector& maxval) { CopyVec(m_maxval[ifab], maxval); } diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index f6c44c777ea..d5dd67226b7 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -28,7 +28,7 @@ class BTDiagnostics final : public Diagnostics { public: - BTDiagnostics (int i, std::string name); + BTDiagnostics (int i, const std::string& name); private: /** Whether to plot raw (i.e., NOT cell-centered) fields */ @@ -176,9 +176,6 @@ private: /** Vector of lab-frame time corresponding to each snapshot */ amrex::Vector m_t_lab; - /** Vector of user-defined physical region for diagnostics in lab-frame - * for each back-transformed snapshot */ - amrex::Vector m_snapshot_domain_lab; /** Vector of physical region corresponding to the buffer that spans a part * of the full back-transformed snapshot */ amrex::Vector m_buffer_domain_lab; @@ -375,26 +372,26 @@ private: /** Interleave lab-frame meta-data of the buffers to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveBufferAndSnapshotHeader ( std::string buffer_Header, - std::string snapshot_Header); + void InterleaveBufferAndSnapshotHeader (const std::string& buffer_Header, + const std::string& snapshot_Header); /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename); + void InterleaveFabArrayHeader (const std::string& Buffer_FabHeader_path, + const std::string& snapshot_FabHeader_path, + const std::string& newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data */ - void InterleaveSpeciesHeader(std::string buffer_species_Header_path, - std::string snapshot_species_Header_path, - std::string species_name, int new_data_index); + void InterleaveSpeciesHeader(const std::string& buffer_species_Header_path, + const std::string& snapshot_species_Header_path, + const std::string& species_name, int new_data_index); /** Interleave lab-frame metadata of the particle header file in the buffers to * be consistent with the merged plotfile lab-frame data */ - void InterleaveParticleDataHeader( std::string buffer_ParticleHdrFilename, - std::string snapshot_ParticleHdrFilename); + void InterleaveParticleDataHeader(const std::string& buffer_ParticleHdrFilename, + const std::string& snapshot_ParticleHdrFilename); /** Initialize particle functors for each species to compute the back-transformed lab-frame data. */ void InitializeParticleFunctors () override; diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index fd9c5c783c1..1cee9909226 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -14,6 +14,7 @@ #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" #include "ComputeDiagFunctors/BackTransformParticleFunctor.H" +#include "FieldSolver/Fields.H" #include "Utils/Algorithms/IsIn.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" @@ -46,14 +47,15 @@ #include using namespace amrex::literals; +using namespace warpx::fields; namespace { constexpr int permission_flag_rwxrxrx = 0755; } -BTDiagnostics::BTDiagnostics (int i, std::string name) - : Diagnostics(i, name) +BTDiagnostics::BTDiagnostics (int i, const std::string& name) + : Diagnostics{i, name} { ReadParameters(); } @@ -565,23 +567,23 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "By" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -596,7 +598,7 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -655,7 +657,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point @@ -681,23 +683,23 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_cell_center_functors_at_lev_size = static_cast(m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } @@ -1290,8 +1292,8 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } void -BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_path, - std::string snapshot_Header_path) +BTDiagnostics::InterleaveBufferAndSnapshotHeader ( const std::string& buffer_Header_path, + const std::string& snapshot_Header_path) { BTDPlotfileHeaderImpl snapshot_HeaderImpl(snapshot_Header_path); snapshot_HeaderImpl.ReadHeaderData(); @@ -1335,9 +1337,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader(const std::string& Buffer_FabHeader_path, + const std::string& snapshot_FabHeader_path, + const std::string& newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); @@ -1365,9 +1367,9 @@ BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, } void -BTDiagnostics::InterleaveSpeciesHeader(std::string buffer_species_Header_path, - std::string snapshot_species_Header_path, - std::string species_name, const int new_data_index) +BTDiagnostics::InterleaveSpeciesHeader(const std::string& buffer_species_Header_path, + const std::string& snapshot_species_Header_path, + const std::string& species_name, const int new_data_index) { BTDSpeciesHeaderImpl BufferSpeciesHeader(buffer_species_Header_path, species_name); @@ -1389,8 +1391,8 @@ BTDiagnostics::InterleaveSpeciesHeader(std::string buffer_species_Header_path, } void -BTDiagnostics::InterleaveParticleDataHeader(std::string buffer_ParticleHdrFilename, - std::string snapshot_ParticleHdrFilename) +BTDiagnostics::InterleaveParticleDataHeader(const std::string& buffer_ParticleHdrFilename, + const std::string& snapshot_ParticleHdrFilename) { BTDParticleDataHeaderImpl BufferParticleHeader(buffer_ParticleHdrFilename); BufferParticleHeader.ReadHeader(); diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 60d184c30e2..3e5fc1f19eb 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -23,7 +23,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - BoundaryScrapingDiagnostics (int i, std::string name); + BoundaryScrapingDiagnostics (int i, const std::string& name); private: /** Read relevant parameters for BoundaryScraping */ diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 11ffce02f09..da1e5fdcc00 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -21,8 +21,8 @@ using namespace amrex::literals; -BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, std::string name) - : Diagnostics(i, name) +BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name) + : Diagnostics{i, name} { ReadParameters(); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index c4809df11a3..199b785096d 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -58,15 +59,22 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const const bool interpolate = true; std::unique_ptr< amrex::MultiFab > slice = nullptr; const int scomp = 0; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + // Generate slice of the cell-centered multifab containing boosted-frame field-data // at current z-boost location for the ith buffer - slice = amrex::get_slice_data(moving_window_dir, - m_current_z_boost[i_buffer], - *m_mf_src, - geom, - scomp, - m_mf_src->nComp(), - interpolate); + slice = amrex::get_slice_data( + moving_window_dir, + m_current_z_boost[i_buffer], + *m_mf_src, + geom, + scomp, + m_mf_src->nComp(), + interpolate); + + // Perform in-place Lorentz-transform of all the fields stored in the slice. LorentzTransformZ( *slice, gamma_boost, beta_boost); diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index 8a6f0765664..9c2d10ad819 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -90,7 +90,7 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst const amrex::Real dt = warpx.getdt(0); for (WarpXParIter pti(*m_pc_src, lev); pti.isValid(); ++pti) { - auto ptile_dst = pc_dst.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex() ); + pc_dst.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex() ); } auto& particles = m_pc_src->GetParticles(lev); diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt index 2ef6af16bfe..5e0eeaab73a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -6,6 +6,7 @@ foreach(D IN LISTS WarpX_DIMS) DivBFunctor.cpp DivEFunctor.cpp JFunctor.cpp + JdispFunctor.cpp RhoFunctor.cpp PartPerCellFunctor.cpp PartPerGridFunctor.cpp diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp index fe503713141..8c2af223304 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp @@ -3,6 +3,7 @@ #include "WarpX.H" #include +#include #include #include @@ -17,6 +18,11 @@ void CellCenterFunctor::operator()(amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { auto& warpx = WarpX::GetInstance(); - InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), - m_convertRZmodes2cartesian); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + + InterpolateMFForDiag( + mf_dst, *m_mf_src, dcomp, + warpx.DistributionMap(m_lev),m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index 3080b500712..d9f9a1e82e0 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -19,6 +19,8 @@ public: to the output diagnostic MultiFab, mf_dst. * \param[in] convertRZmodes2cartesian (in cylindrical) whether to * sum all modes in mf_src before cell-centering into dst multifab. + * \param[in] deposit_current whether to deposit current. Used to output current in cases + where no field solve is used. * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ JFunctor (int dir, int lev, diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 9938cf095bd..ebaec47b2f1 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -6,13 +6,17 @@ #include "JFunctor.H" +#include "FieldSolver/Fields.H" #include "Particles/MultiParticleContainer.H" #include "WarpX.H" #include +#include #include #include +using namespace warpx::fields; + JFunctor::JFunctor (const int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, @@ -27,7 +31,7 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ { auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* m_mf_src = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) @@ -36,16 +40,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getField(FieldType::current_fp, m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getField(FieldType::current_fp, m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getField(FieldType::current_fp, m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); @@ -60,6 +64,10 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ } } - InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), - m_convertRZmodes2cartesian); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + + InterpolateMFForDiag( + mf_dst, *m_mf_src, dcomp, + warpx.DistributionMap(m_lev), m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H new file mode 100644 index 00000000000..aa364f6da3b --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H @@ -0,0 +1,49 @@ +#ifndef WARPX_JDISPFUNCTOR_H_ +#define WARPX_JDISPFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +#include + +/** + * \brief Functor to cell-center MF for displacement current density and store + * result in mf_out. + */ +class +JdispFunctor final: public ComputeDiagFunctor +{ +public: + /** + * + * \param[in] dir direction of vector field to operate on + * \param[in] lev level of multifab. Used for averaging in rz. + * \param[in] crse_ratio for interpolating field values from the simulation Multifab, src_mf, + * to the output diagnostic MultiFab, mf_dst. + * \param[in] convertRZmodes2cartesian (in cylindrical) whether to + * sum all modes in mf_src before cell-centering into dst multifab. + * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. + */ + JdispFunctor (int dir, int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1); + /** \brief Cell-center m_mf_src and write the result in mf_dst. + * + * In cylindrical geometry, by default this functor average all components + * of a MultiFab and writes into one single component. + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered + * data is stored + */ + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; +private: + /** direction of the electron current density to save */ + int m_dir; + /** level on which mf_src is defined */ + int m_lev; + /** (for cylindrical) whether to average all modes into 1 comp */ + bool m_convertRZmodes2cartesian; +}; + +#endif // WARPX_JDISPFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp new file mode 100644 index 00000000000..aac5869da65 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -0,0 +1,111 @@ +/* This file is part of Warpx. + * + * Authors: Avigdor Veksler + * License: BSD-3-Clause-LBNL +*/ +#include "JdispFunctor.H" + +#include "WarpX.H" +#include "FieldSolver/Fields.H" +#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" +#include "Particles/MultiParticleContainer.H" + +#include +#include +#include +#include + +using namespace amrex; +using namespace warpx::fields; + +JdispFunctor::JdispFunctor (int dir, int lev, + amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) + : ComputeDiagFunctor(ncomp, crse_ratio), m_dir(dir), m_lev(lev), + m_convertRZmodes2cartesian(convertRZmodes2cartesian) +{ } + +void +JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const +{ + auto& warpx = WarpX::GetInstance(); + auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); + + /** pointer to total simulation current (J) multifab */ + amrex::MultiFab* mf_j = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, + "Displacement current diagnostic is only implemented for the HybridPICModel."); + AMREX_ASSUME(hybrid_pic_model != nullptr); + + /** pointer to current calculated from Ampere's Law (Jamp) multifab */ + amrex::MultiFab* mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir);; + + //if (!hybrid_pic_model) { + // To finish this implementation, we need to implement a method to + // calculate (∇ x B). + + // Skeleton for future implementation for solvers other than HybridPIC. + // Get curlB multifab + + // Divide curlB multifab by mu0 to get units of current + // mf_curlB->mult(1.0/PhysConsts::mu0) + //} + + // A Jdisp multifab is generated to hold displacement current. + amrex::MultiFab Jdisp( mf_j->boxArray(), mf_j->DistributionMap(), 1, mf_j->nGrowVect() ); + Jdisp.setVal(0); + + // J_displacement = curl x B / mu0 - J + amrex::MultiFab::LinComb( + Jdisp, 1, *mf_curlB, 0, + -1, *mf_j, 0, 0, 1, Jdisp.nGrowVect() + ); + + if (hybrid_pic_model) { + // Subtract the interpolated j_external value from j_displacement. + /** pointer to external currents (Jext) multifab */ + amrex::MultiFab* mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + + // Index type required for interpolating Jext from their respective + // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement + // locations. The staggering of J_displacement is the same as the + // staggering for J, so we use J_stag as the interpolation map. + // For interp to work below, the indices of the undefined dimensions + // must match. We set them as (1,1,1). + amrex::GpuArray Jext_IndexType = {1, 1, 1}; + amrex::GpuArray J_IndexType = {1, 1, 1}; + amrex::IntVect Jext_stag = mf_j_external->ixType().toIntVect(); + amrex::IntVect J_stag = mf_j->ixType().toIntVect(); + + // Index types for the dimensions simulated are overwritten. + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + Jext_IndexType[idim] = Jext_stag[idim]; + J_IndexType[idim] = J_stag[idim]; + } + + // Parameters for `interp` that maps from Jext to J. + // The "coarsening is just 1 i.e. no coarsening" + amrex::GpuArray const& coarsen = {1, 1, 1}; + + // Loop through the grids, and over the tiles within each grid to + // subtract the interpolated Jext from J_displacement. +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(Jdisp, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + + Array4 const& Jdisp_arr = Jdisp.array(mfi); + Array4 const& Jext = mf_j_external->const_array(mfi); + + // Loop over cells and update the Jdisp MultiFab + amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Interpolate Jext to the staggering of J + auto const jext_interp = ablastr::coarsen::sample::Interp(Jext, Jext_IndexType, J_IndexType, coarsen, i, j, k, 0); + Jdisp_arr(i, j, k, 0) -= jext_interp; + }); + } + } + + InterpolateMFForDiag(mf_dst, Jdisp, dcomp, warpx.DistributionMap(m_lev), + m_convertRZmodes2cartesian); +} diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index 9c329263b0f..fd1624b8708 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -4,6 +4,7 @@ CEXE_sources += PartPerGridFunctor.cpp CEXE_sources += DivBFunctor.cpp CEXE_sources += DivEFunctor.cpp CEXE_sources += JFunctor.cpp +CEXE_sources += JdispFunctor.cpp CEXE_sources += RhoFunctor.cpp CEXE_sources += BackTransformFunctor.cpp CEXE_sources += BackTransformParticleFunctor.cpp diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 520951218f9..7f141e95b32 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -30,9 +30,9 @@ public: * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ ParticleReductionFunctor(const amrex::MultiFab * mf_src, int lev, - amrex::IntVect crse_ratio, std::string fn_str, + amrex::IntVect crse_ratio, const std::string& fn_str, int ispec, bool do_average, - bool do_filter, std::string filter_str, + bool do_filter, const std::string& filter_str, int ncomp=1); /** \brief Compute the average of the function m_map_fn over each grid cell. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 2be9d2ffb36..1f7a8fc181b 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -18,9 +18,9 @@ using namespace amrex::literals; ParticleReductionFunctor::ParticleReductionFunctor (const amrex::MultiFab* mf_src, const int lev, - const amrex::IntVect crse_ratio, const std::string fn_str, + const amrex::IntVect crse_ratio, const std::string& fn_str, const int ispec, const bool do_average, - const bool do_filter, const std::string filter_str, const int ncomp) + const bool do_filter, const std::string& filter_str, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), m_ispec(ispec), m_do_average(do_average), m_do_filter(do_filter) { // mf_src will not be used, let's make sure it's null. @@ -52,8 +52,6 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, constexpr int ng = 1; // Temporary cell-centered, multi-component MultiFab for storing particles per cell. amrex::MultiFab red_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); - // Set value to 0, and increment the value in each cell with ppc. - red_mf.setVal(0._rt); auto& pc = warpx.GetPartContainer().GetParticleContainer(m_ispec); auto pcomps = pc.getParticleComps(); const int iupstream = pcomps["upstream"] - PIdx::nattribs; @@ -105,7 +103,6 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, }); if (m_do_average) { amrex::MultiFab ppc_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); - ppc_mf.setVal(0._rt); // Add the weight for each particle -- total number of particles of this species ParticleToMesh(pc, ppc_mf, m_lev, [=] AMREX_GPU_DEVICE (const WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType& ptd, diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index 65741e4ff20..be1322bd61b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -5,7 +5,6 @@ #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/MultiParticleContainer.H" -#include "WarpX.H" class FlushFormat { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index b083e60529f..27d0ad84b9c 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -5,6 +5,7 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FieldSolver/Fields.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" @@ -19,6 +20,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { @@ -62,85 +64,85 @@ FlushFormatCheckpoint::WriteToFile ( for (int lev = 0; lev < nlev; ++lev) { - VisMF::Write(warpx.getEfield_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_fp")); - VisMF::Write(warpx.getEfield_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); - VisMF::Write(warpx.getEfield_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getEfield_avg_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_fp")); - VisMF::Write(warpx.getEfield_avg_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_fp")); - VisMF::Write(warpx.getEfield_avg_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_fp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getcurrent_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(warpx.getcurrent_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(warpx.getcurrent_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Write(warpx.getEfield_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_cp")); - VisMF::Write(warpx.getEfield_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_cp")); - VisMF::Write(warpx.getEfield_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getEfield_avg_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_cp")); - VisMF::Write(warpx.getEfield_avg_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_cp")); - VisMF::Write(warpx.getEfield_avg_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_cp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getcurrent_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_cp")); - VisMF::Write(warpx.getcurrent_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_cp")); - VisMF::Write(warpx.getcurrent_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_cp")); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 5774022787c..69dfc740063 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -1,16 +1,17 @@ #include "FlushFormatPlotfile.H" -#include "Particles/ParticleIO.H" +#include "FieldSolver/Fields.H" +#include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/Filter/FilterFunctors.H" #include "Particles/WarpXParticleContainer.H" +#include "Particles/ParticleIO.H" #include "Particles/PinnedMemoryParticleContainer.H" #include "Utils/Interpolate.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" -#include "Diagnostics/MultiDiagnostics.H" #include #include @@ -47,6 +48,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { @@ -481,7 +483,7 @@ WriteZeroRawMF( const MultiFab& F, const DistributionMapping& dm, * coarse and fine patch to have the same shape. */ void -WriteCoarseVector( const std::string field_name, +WriteCoarseVector( const std::string& field_name, const MultiFab* Fx_cp, const MultiFab* Fy_cp, const MultiFab* Fz_cp, @@ -521,7 +523,7 @@ WriteCoarseVector( const std::string field_name, * coarse and fine patch to have the same shape. */ void -WriteCoarseScalar( const std::string field_name, +WriteCoarseScalar( const std::string& field_name, const MultiFab* F_cp, const MultiFab* F_fp, const DistributionMapping& dm, @@ -564,84 +566,84 @@ FlushFormatPlotfile::WriteAllRawFields( // Auxiliary patch - WriteRawMF( warpx.getEfield(lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield(lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield(lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch - WriteRawMF( warpx.getEfield_fp(lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield_fp(lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield_fp(lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); - if (warpx.get_pointer_F_fp(lev)) + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); + if (warpx.isFieldInitialized(FieldType::F_fp, lev)) { - WriteRawMF(warpx.getF_fp(lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); + WriteRawMF(warpx.getField(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); } - if (warpx.get_pointer_rho_fp(lev)) + if (warpx.isFieldInitialized(FieldType::rho_fp, lev)) { // rho_fp will have either ncomps or 2*ncomps (2 being the old and new). When 2, return the new so // there is time synchronization. - const int nstart = warpx.getrho_fp(lev).nComp() - WarpX::ncomps; - const MultiFab rho_new(warpx.getrho_fp(lev), amrex::make_alias, nstart, WarpX::ncomps); + const int nstart = warpx.getField(FieldType::rho_fp, lev).nComp() - WarpX::ncomps; + const MultiFab rho_new(warpx.getField(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } - if (warpx.get_pointer_phi_fp(lev) != nullptr) { - WriteRawMF(warpx.getphi_fp(lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + if (warpx.isFieldInitialized(FieldType::phi_fp, lev)) { + WriteRawMF(warpx.getField(FieldType::phi_fp, lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); } // Averaged fields on fine patch if (WarpX::fft_do_time_averaging) { - WriteRawMF(warpx.getEfield_avg_fp(lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, "Ex_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getEfield_avg_fp(lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, "Ey_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getEfield_avg_fp(lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, "Ez_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, "Bx_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, "By_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, "Bz_avg_fp", lev, plot_raw_fields_guards); } // Coarse path if (lev > 0) { WriteCoarseVector( "E", - warpx.get_pointer_Efield_cp(lev, 0), warpx.get_pointer_Efield_cp(lev, 1), warpx.get_pointer_Efield_cp(lev, 2), - warpx.get_pointer_Efield_fp(lev, 0), warpx.get_pointer_Efield_fp(lev, 1), warpx.get_pointer_Efield_fp(lev, 2), + warpx.getFieldPointer(FieldType::Efield_cp, lev, 0), warpx.getFieldPointer(FieldType::Efield_cp, lev, 1), warpx.getFieldPointer(FieldType::Efield_cp, lev, 2), + warpx.getFieldPointer(FieldType::Efield_fp, lev, 0), warpx.getFieldPointer(FieldType::Efield_fp, lev, 1), warpx.getFieldPointer(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", - warpx.get_pointer_Bfield_cp(lev, 0), warpx.get_pointer_Bfield_cp(lev, 1), warpx.get_pointer_Bfield_cp(lev, 2), - warpx.get_pointer_Bfield_fp(lev, 0), warpx.get_pointer_Bfield_fp(lev, 1), warpx.get_pointer_Bfield_fp(lev, 2), + warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), + warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.get_pointer_current_cp(lev, 0), warpx.get_pointer_current_cp(lev, 1), warpx.get_pointer_current_cp(lev, 2), - warpx.get_pointer_current_fp(lev, 0), warpx.get_pointer_current_fp(lev, 1), warpx.get_pointer_current_fp(lev, 2), + warpx.getFieldPointer(FieldType::current_cp, lev, 0), warpx.getFieldPointer(FieldType::current_cp, lev, 1), warpx.getFieldPointer(FieldType::current_cp, lev, 2), + warpx.getFieldPointer(FieldType::current_fp, lev, 0), warpx.getFieldPointer(FieldType::current_fp, lev, 1), warpx.getFieldPointer(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); - if (warpx.get_pointer_F_fp(lev) && warpx.get_pointer_F_cp(lev)) + if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) { - WriteCoarseScalar("F", warpx.get_pointer_F_cp(lev), warpx.get_pointer_F_fp(lev), + WriteCoarseScalar("F", warpx.getFieldPointer(FieldType::F_cp, lev), warpx.getFieldPointer(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } - if (warpx.get_pointer_rho_fp(lev) && warpx.get_pointer_rho_cp(lev)) + if (warpx.isFieldInitialized(FieldType::rho_fp, lev) && warpx.isFieldInitialized(FieldType::rho_cp, lev)) { // Use the component 1 of `rho_cp`, i.e. rho_new for time synchronization - WriteCoarseScalar("rho", warpx.get_pointer_rho_cp(lev), warpx.get_pointer_rho_fp(lev), + WriteCoarseScalar("rho", warpx.getFieldPointer(FieldType::rho_cp, lev), warpx.getFieldPointer(FieldType::rho_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 1); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index d2ec9a5a4e0..45ea40077e4 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -45,7 +45,7 @@ public: * \param[in] amr_mesh an AmrMesh instance * \param[in] diag_name ParmParse scope string. */ - FlushFormatSensei (amrex::AmrMesh *amr_mesh, std::string diag_name); + FlushFormatSensei (amrex::AmrMesh *amr_mesh, const std::string& diag_name); /** Do in-situ visualization for field and particle data */ void WriteToFile ( diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index 348e1da4a00..468ed81ce18 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -8,7 +8,7 @@ #endif FlushFormatSensei::FlushFormatSensei (amrex::AmrMesh *amr_mesh, - std::string diag_name) : + const std::string& diag_name) : m_amr_mesh(amr_mesh) { #ifndef AMREX_USE_SENSEI_INSITU diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 42b60c213f2..1b999a9b361 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -9,7 +9,7 @@ class FullDiagnostics final : public Diagnostics { public: - FullDiagnostics (int i, std::string name); + FullDiagnostics (int i, const std::string& name); private: /** Read user-requested parameters for full diagnostics */ void ReadParameters (); diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index d93c67fe893..b25f899e29d 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -1,14 +1,17 @@ #include "FullDiagnostics.H" + #include "ComputeDiagFunctors/CellCenterFunctor.H" #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" #include "ComputeDiagFunctors/JFunctor.H" +#include "ComputeDiagFunctors/JdispFunctor.H" #include "ComputeDiagFunctors/PartPerCellFunctor.H" #include "ComputeDiagFunctors/PartPerGridFunctor.H" #include "ComputeDiagFunctors/ParticleReductionFunctor.H" #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FieldSolver/Fields.H" #include "FlushFormats/FlushFormat.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/IsIn.H" @@ -39,8 +42,9 @@ #include using namespace amrex::literals; +using namespace warpx::fields; -FullDiagnostics::FullDiagnostics (int i, std::string name): +FullDiagnostics::FullDiagnostics (int i, const std::string& name): Diagnostics{i, name}, m_solver_deposits_current{ (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None) || @@ -169,15 +173,15 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -210,37 +214,37 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Er"), ncomp); } } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Et"), ncomp); } } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Ez"), ncomp); } } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Br"), ncomp); } } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bt"), ncomp); } } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bz"), ncomp); @@ -266,6 +270,24 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) if (update_varnames) { AddRZModesToOutputNames(std::string("jz"), ncomp); } + } else if ( m_varnames_fields[comp] == "jr_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jr_displacement"), ncomp); + } + } else if ( m_varnames_fields[comp] == "jt_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jt_displacement"), ncomp); + } + } else if ( m_varnames_fields[comp] == "jz_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jz_displacement"), ncomp); + } } else if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, @@ -282,19 +304,19 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } i++; } else if ( m_varnames_fields[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_G_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); } } else if ( m_varnames_fields[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_phi_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("phi"), ncomp); @@ -310,14 +332,16 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) m_varnames.push_back(std::string("part_per_grid")); } } else if ( m_varnames_fields[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio, - false, ncomp); + m_all_field_functors[lev][comp] = std::make_unique( + warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), + lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divB"), ncomp); } } else if ( m_varnames_fields[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio, - false, ncomp); + m_all_field_functors[lev][comp] = std::make_unique( + warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divE"), ncomp); } @@ -363,15 +387,15 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); } // Check if divE is requested @@ -390,8 +414,6 @@ FullDiagnostics::AddRZModesToDiags (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; - // First index of m_all_field_functors[lev] where RZ modes are stored - auto icomp =static_cast(m_all_field_functors[0].size()); const std::array coord {"r", "theta", "z"}; // Er, Etheta, Ez, Br, Btheta, Bz, jr, jtheta, jz @@ -403,48 +425,45 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (rho_requested) { n_new_fields += 1; } - m_all_field_functors[lev].resize( m_all_field_functors[0].size() + n_new_fields ); + m_all_field_functors[lev].reserve( m_all_field_functors[0].size() + n_new_fields ); // E for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Efield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.get_pointer_Efield_aux(0, 0)->nComp()); - icomp += 1; + warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp()); } // B for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Bfield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.get_pointer_Bfield_aux(0, 0)->nComp()); - icomp += 1; + warpx.getFieldPointer(FieldType::Bfield_aux, 0, 0)->nComp()); } // j for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; - icomp += 1; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.get_pointer_current_fp(0, 0)->nComp()); + warpx.getFieldPointer(FieldType::current_fp, 0, 0)->nComp()); } // divE if (divE_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, - m_crse_ratio, false, ncomp_multimodefab); - icomp += 1; + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); } // rho if (rho_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp_multimodefab); - icomp += 1; + m_all_field_functors[lev].push_back(std::make_unique( + lev, m_crse_ratio, true, -1, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("rho"), ncomp_multimodefab); } // Sum the number of components in input vector m_all_field_functors @@ -630,14 +649,16 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if ( m_varnames[comp] == "jz_displacement" ) { + m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); @@ -646,65 +667,74 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_G_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_phi_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_grid" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Efield_aux, lev), lev, m_crse_ratio); } else { #ifdef WARPX_DIM_RZ if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jr" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; } else if ( m_varnames[comp] == "jt" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if (m_varnames[comp] == "jr_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true); + } else if (m_varnames[comp] == "jt_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); } else { WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } #else // Valid transverse fields in Cartesian coordinates if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jx" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; } else if ( m_varnames[comp] == "jy" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if ( m_varnames[comp] == "jx_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jy_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); } else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); } #endif diff --git a/Source/Diagnostics/OpenPMDHelpFunction.cpp b/Source/Diagnostics/OpenPMDHelpFunction.cpp index 6170249b52b..cc798adc29e 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.cpp +++ b/Source/Diagnostics/OpenPMDHelpFunction.cpp @@ -6,8 +6,11 @@ * License: BSD-3-Clause-LBNL */ #include "OpenPMDHelpFunction.H" + #include "Utils/TextMsg.H" +#include + std::string WarpXOpenPMDFileType () { diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.H b/Source/Diagnostics/ParticleDiag/ParticleDiag.H index 14a146d7c01..cefce3c1b23 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.H +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.H @@ -17,11 +17,13 @@ class ParticleDiag { public: - ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); + ParticleDiag(const std::string& diag_name, const std::string& name, + WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); [[nodiscard]] WarpXParticleContainer* getParticleContainer() const { return m_pc; } [[nodiscard]] PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } [[nodiscard]] std::string getSpeciesName() const { return m_name; } amrex::Vector m_plot_flags; + bool m_plot_phi = false; // Whether to output the potential phi on the particles bool m_do_random_filter = false; bool m_do_uniform_filter = false; diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index fd1b9e7ec3a..3672557ede6 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -6,6 +6,8 @@ #include "Utils/TextMsg.H" #include "WarpX.H" +#include + #include #include @@ -13,8 +15,10 @@ using namespace amrex; -ParticleDiag::ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc) - : m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) +ParticleDiag::ParticleDiag ( + const std::string& diag_name, const std::string& name, + WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc): + m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) { //variable to set m_plot_flags size const int plot_flag_size = pc->NumRealComps(); @@ -26,21 +30,45 @@ ParticleDiag::ParticleDiag(std::string diag_name, std::string name, WarpXParticl amrex::Vector variables; const int variables_specified = pp_diag_name_species_name.queryarr("variables", variables); - if (variables_specified){ + if (variables_specified) { // If only specific variables have been specified, fill m_plot_flags with zero and only set // requested variables to one std::fill(m_plot_flags.begin(), m_plot_flags.end(), 0); + bool contains_positions = false; if (variables[0] != "none"){ - const std::map existing_variable_names = pc->getParticleComps(); + std::map existing_variable_names = pc->getParticleComps(); +#ifdef WARPX_DIM_RZ + // we reconstruct to Cartesian x,y,z for RZ particle output + existing_variable_names["y"] = PIdx::theta; +#endif for (const auto& var : variables){ - const auto search = existing_variable_names.find(var); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - search != existing_variable_names.end(), - "variables argument '" + var - +"' is not an existing attribute for this species"); - m_plot_flags[existing_variable_names.at(var)] = 1; + if (var == "phi") { + // User requests phi on particle. This is *not* part of the variables that + // the particle container carries, and is only added to particles during output. + // Therefore, this case needs to be treated specifically. + m_plot_phi = true; + } else { + const auto search = existing_variable_names.find(var); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + search != existing_variable_names.end(), + "variables argument '" + var + +"' is not an existing attribute for this species"); + m_plot_flags[existing_variable_names.at(var)] = 1; + + if (var == "x" || var == "y" || var == "z") { + contains_positions = true; + } + } } } + + if (!contains_positions) { + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + diag_name + "." + name + ".variables contains no particle positions!", + ablastr::warn_manager::WarnPriority::high + ); + } } #ifdef WARPX_DIM_RZ diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index a8bb9303fe1..f1f4d426a4a 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -7,6 +7,7 @@ * License: BSD-3-Clause-LBNL */ +#include "FieldSolver/Fields.H" #include "Particles/ParticleIO.H" #include "Particles/MultiParticleContainer.H" #include "Particles/PhysicalParticleContainer.H" @@ -41,6 +42,7 @@ #include using namespace amrex; +using namespace warpx::fields; void LaserParticleContainer::ReadHeader (std::istream& is) @@ -235,3 +237,58 @@ MultiParticleContainer::WriteHeader (std::ostream& os) const allcontainers.at(i)->WriteHeader(os); } } + +void +storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, + int electrostatic_solver_id, bool is_full_diagnostic ) { + + using PinnedParIter = typename PinnedMemoryParticleContainer::ParIterType; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); + // When this is not a full diagnostic, the particles are not written at the same physical time (i.e. PIC iteration) + // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). + // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. + // To avoid confusion, we raise an error in this case. + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + is_full_diagnostic, + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available with `diag_type = Full`."); + tmp.AddRealComp("phi"); + int const phi_index = tmp.getParticleComps().at("phi"); + auto& warpx = WarpX::GetInstance(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (int lev=0; lev<=warpx.finestLevel(); lev++) { + const amrex::Geometry& geom = warpx.Geom(lev); + auto plo = geom.ProbLoArray(); + auto dxi = geom.InvCellSizeArray(); + amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + + for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { + + auto phi_grid = phi[pti].array(); + const auto getPosition = GetParticlePosition(pti); + amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); + + // Loop over the particles and update their position + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long ip) { + + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, i, j, k, W); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); + phi_particle_arr[ip] = phi_value; + } + ); + } + } +} diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.H b/Source/Diagnostics/ReducedDiags/BeamRelevant.H index 018ddef5463..ed410c72fc9 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.H +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.H @@ -23,7 +23,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - BeamRelevant(std::string rd_name); + BeamRelevant(const std::string& rd_name); /// name of beam species std::string m_beam_name; diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index fbe67ba6fc5..ae7d5230b4c 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -31,7 +32,7 @@ using namespace amrex; // constructor -BeamRelevant::BeamRelevant (std::string rd_name) +BeamRelevant::BeamRelevant (const std::string& rd_name) : ReducedDiags{rd_name} { // read beam name @@ -198,14 +199,14 @@ void BeamRelevant::ComputeDiags (int step) using PType = typename WarpXParticleContainer::SuperParticleType; - amrex::ReduceOps reduce_ops; - auto r = amrex::ParticleReduce>( + // number of reduction operations in first concurrent batch + constexpr size_t num_red_ops_1 = 8; + TypeMultiplier reduce_ops_1; + using ReducedDataT1 = TypeMultiplier; + + auto r1 = amrex::ParticleReduce( myspc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple - + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> ReducedDataT1::Type { const ParticleReal p_ux = p.rdata(PIdx::ux); const ParticleReal p_uy = p.rdata(PIdx::uy); @@ -242,18 +243,17 @@ void BeamRelevant::ComputeDiags (int step) p_ux_mean, p_uy_mean, p_uz_mean, p_gm_mean}; }, - reduce_ops); - - std::vector values_per_rank_1st = { - amrex::get<0>(r), // w - amrex::get<1>(r), // x_mean - amrex::get<2>(r), // y_mean - amrex::get<3>(r), // z_mean - amrex::get<4>(r), // ux_mean - amrex::get<5>(r), // uy_mean - amrex::get<6>(r), // uz_mean - amrex::get<7>(r), // gm_mean - }; + reduce_ops_1); + + std::vector values_per_rank_1st(num_red_ops_1); + + /* contains in this order: + * w, x_mean, y_mean, z_mean + * ux_mean, uy_mean, uz_mean, gm_mean + */ + amrex::constexpr_for<0, num_red_ops_1> ([&](auto i) { + values_per_rank_1st[i] = amrex::get(r1); + }); // reduced sum over mpi ranks (allreduce) amrex::ParallelAllReduce::Sum @@ -275,16 +275,15 @@ void BeamRelevant::ComputeDiags (int step) return; } - amrex::ReduceOps reduce_ops2; + // number of reduction operations in second concurrent batch + constexpr size_t num_red_ops_2 = 11; + + TypeMultiplier reduce_ops2; + using ReducedDataT2 = TypeMultiplier; - auto r2 = amrex::ParticleReduce>( + auto r2 = amrex::ParticleReduce( myspc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple - + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> ReducedDataT2::Type { const ParticleReal p_ux = p.rdata(PIdx::ux); const ParticleReal p_uy = p.rdata(PIdx::uy); @@ -336,19 +335,18 @@ void BeamRelevant::ComputeDiags (int step) }, reduce_ops2); - std::vector values_per_rank_2nd = { - amrex::get<0>(r2), // x_ms - amrex::get<1>(r2), // y_ms - amrex::get<2>(r2), // z_ms - amrex::get<3>(r2), // ux_ms - amrex::get<4>(r2), // uy_ms - amrex::get<5>(r2), // uz_ms - amrex::get<6>(r2), // gm_ms - amrex::get<7>(r2), // xux - amrex::get<8>(r2), // yuy - amrex::get<9>(r2), // zuz - amrex::get<10>(r2) // charge - }; + std::vector values_per_rank_2nd(num_red_ops_2); + + /* contains in this order: + * x_ms, y_ms, z_ms + * ux_ms, uy_ms, uz_ms, + * gm_ms + * xux, yuy, zuz, + * charge + */ + amrex::constexpr_for<0, num_red_ops_2> ([&](auto i) { + values_per_rank_2nd[i] = amrex::get(r2); + }); // reduced sum over mpi ranks (reduce to IO rank) ParallelDescriptor::ReduceRealSum diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H index ed69ce01767..3cea5b0b30e 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H @@ -33,7 +33,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ChargeOnEB (std::string rd_name); + ChargeOnEB (const std::string& rd_name); /** * This function computes the charge at the surface of the EB: diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index c6f75e4a712..2991831420e 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -8,6 +8,7 @@ #include "ChargeOnEB.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/Parser/ParserUtils.H" @@ -26,9 +27,10 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor -ChargeOnEB::ChargeOnEB (std::string rd_name) +ChargeOnEB::ChargeOnEB (const std::string& rd_name) : ReducedDiags{rd_name} { // Only 3D is working for now @@ -93,9 +95,9 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev - const amrex::MultiFab & Ex = warpx.getEfield_fp(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield_fp(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield_fp(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_fp, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_fp, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_fp, lev,2); // get EB structures amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H index fe00dd2e7e3..3b93eb1ed25 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H @@ -26,7 +26,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ColliderRelevant(std::string rd_name); + ColliderRelevant(const std::string& rd_name); /// name of the two colliding species std::vector m_beam_name; diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 4cd1a66f3f2..434ab1d7949 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -8,6 +8,7 @@ #include "ColliderRelevant.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif @@ -58,9 +59,10 @@ #include using namespace amrex; +using namespace warpx::fields; -ColliderRelevant::ColliderRelevant (std::string rd_name) -: ReducedDiags{std::move(rd_name)} +ColliderRelevant::ColliderRelevant (const std::string& rd_name) +: ReducedDiags{rd_name} { // read colliding species names - must be 2 const amrex::ParmParse pp_rd_name(m_rd_name); @@ -441,12 +443,12 @@ void ColliderRelevant::ComputeDiags (int step) // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::MultiFab & Ex = warpx.getEfield(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield(lev,2); - const amrex::MultiFab & Bx = warpx.getBfield(lev,0); - const amrex::MultiFab & By = warpx.getBfield(lev,1); - const amrex::MultiFab & Bz = warpx.getBfield(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.H b/Source/Diagnostics/ReducedDiags/FieldEnergy.H index 4f5d1656841..40de174526e 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.H +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.H @@ -26,7 +26,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldEnergy(std::string rd_name); + FieldEnergy(const std::string& rd_name); /** * This function computes the field energy (EF): diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index a88690f863f..f4b4e2a39a1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -7,6 +7,7 @@ #include "FieldEnergy.H" +#include "FieldSolver/Fields.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -28,9 +29,10 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor -FieldEnergy::FieldEnergy (std::string rd_name) +FieldEnergy::FieldEnergy (const std::string& rd_name) : ReducedDiags{rd_name} { @@ -89,12 +91,12 @@ void FieldEnergy::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // get cell size Geometry const & geom = warpx.Geom(lev); diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.H b/Source/Diagnostics/ReducedDiags/FieldMaximum.H index 95ae6ff16da..1d7cf4d236c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldMaximum(std::string rd_name); + FieldMaximum(const std::string& rd_name); /** * This function computes the maximum value of Ex, Ey, Ez, |E|, Bx, By, Bz and |B| diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 6465ee2be1b..24d9dda3ea6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -7,6 +7,7 @@ #include "FieldMaximum.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -38,9 +39,10 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor -FieldMaximum::FieldMaximum (std::string rd_name) +FieldMaximum::FieldMaximum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working @@ -114,12 +116,12 @@ void FieldMaximum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); constexpr int noutputs = 8; // max of Ex,Ey,Ez,|E|,Bx,By,Bz and |B| constexpr int index_Ex = 0; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.H b/Source/Diagnostics/ReducedDiags/FieldMomentum.H index 0f04b6413d1..99eabf3772f 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.H @@ -24,7 +24,7 @@ public: * * \param[in] rd_name reduced diags names */ - FieldMomentum(std::string rd_name); + FieldMomentum(const std::string& rd_name); /** * \brief This function computes the electromagnetic momentum, diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index fe2040e50d2..f182acd5ba2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -7,6 +7,7 @@ #include "FieldMomentum.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" @@ -37,8 +38,9 @@ #include using namespace amrex; +using namespace warpx::fields; -FieldMomentum::FieldMomentum (std::string rd_name) +FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working @@ -106,12 +108,12 @@ void FieldMomentum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // Get MultiFab data at given refinement level - const amrex::MultiFab & Ex = warpx.getEfield(lev, 0); - const amrex::MultiFab & Ey = warpx.getEfield(lev, 1); - const amrex::MultiFab & Ez = warpx.getEfield(lev, 2); - const amrex::MultiFab & Bx = warpx.getBfield(lev, 0); - const amrex::MultiFab & By = warpx.getBfield(lev, 1); - const amrex::MultiFab & Bz = warpx.getBfield(lev, 2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev, 0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev, 1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev, 2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev, 1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); // Cell-centered index type const amrex::GpuArray cc{0,0,0}; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.H b/Source/Diagnostics/ReducedDiags/FieldProbe.H index 9f318c4f8a0..a7c04649999 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.H @@ -41,7 +41,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldProbe (std::string rd_name); + FieldProbe (const std::string& rd_name); /** * This function assins test/data particles to constructed environemnt diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 24ad0e64ea8..7364d5989f1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -7,6 +7,7 @@ #include "FieldProbe.H" #include "FieldProbeParticleContainer.H" +#include "FieldSolver/Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/UpdatePosition.H" @@ -44,10 +45,11 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor -FieldProbe::FieldProbe (std::string rd_name) +FieldProbe::FieldProbe (const std::string& rd_name) : ReducedDiags{rd_name}, m_probe(&WarpX::GetInstance()) { @@ -391,12 +393,12 @@ void FieldProbe::ComputeDiags (int step) } // get MultiFab data at lev - const amrex::MultiFab &Ex = warpx.getEfield(lev, 0); - const amrex::MultiFab &Ey = warpx.getEfield(lev, 1); - const amrex::MultiFab &Ez = warpx.getEfield(lev, 2); - const amrex::MultiFab &Bx = warpx.getBfield(lev, 0); - const amrex::MultiFab &By = warpx.getBfield(lev, 1); - const amrex::MultiFab &Bz = warpx.getBfield(lev, 2); + const amrex::MultiFab &Ex = warpx.getField(FieldType::Efield_aux, lev, 0); + const amrex::MultiFab &Ey = warpx.getField(FieldType::Efield_aux, lev, 1); + const amrex::MultiFab &Ez = warpx.getField(FieldType::Efield_aux, lev, 2); + const amrex::MultiFab &Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); + const amrex::MultiFab &By = warpx.getField(FieldType::Bfield_aux, lev, 1); + const amrex::MultiFab &Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); /* * Prepare interpolation of field components to probe_position diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 7e7aecb9167..dc26de3c1e6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -76,6 +76,10 @@ FieldProbeParticleContainer::AddNParticles (int lev, // number of particles to add auto const np = static_cast(x.size()); + if (np <= 0){ + Redistribute(); + return; + } // have to resize here, not in the constructor because grids have not // been built when constructor was called. @@ -122,8 +126,8 @@ FieldProbeParticleContainer::AddNParticles (int lev, pinned_tile.push_back_real(FieldProbePIdx::Bz, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::S, np, 0.0); - auto old_np = particle_tile.numParticles(); - auto new_np = old_np + pinned_tile.numParticles(); + const auto old_np = particle_tile.numParticles(); + const auto new_np = old_np + pinned_tile.numParticles(); particle_tile.resize(new_np); amrex::copyParticles( particle_tile, pinned_tile, 0, old_np, pinned_tile.numParticles()); diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index e99c87ad14f..a15d6489e2c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -9,6 +9,7 @@ #define WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDREDUCTION_H_ #include "ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "WarpX.H" #include @@ -36,7 +37,6 @@ #include #include - /** * This class contains a function that computes an arbitrary reduction of the fields. The function * used in the reduction is defined by an input file parser expression and the reduction operation @@ -50,7 +50,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldReduction(std::string rd_name); + FieldReduction(const std::string& rd_name); /** * This function is called at every time step, and if necessary calls the templated function @@ -99,15 +99,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getEfield(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield(lev,2); - const amrex::MultiFab & Bx = warpx.getBfield(lev,0); - const amrex::MultiFab & By = warpx.getBfield(lev,1); - const amrex::MultiFab & Bz = warpx.getBfield(lev,2); - const amrex::MultiFab & jx = warpx.getcurrent_fp(lev,0); - const amrex::MultiFab & jy = warpx.getcurrent_fp(lev,1); - const amrex::MultiFab & jz = warpx.getcurrent_fp(lev,2); + const amrex::MultiFab & Ex = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & jx = warpx.getField(warpx::fields::FieldType::current_fp, lev,0); + const amrex::MultiFab & jy = warpx.getField(warpx::fields::FieldType::current_fp, lev,1); + const amrex::MultiFab & jz = warpx.getField(warpx::fields::FieldType::current_fp, lev,2); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp index 85e4d4028e6..683ae1921d6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp @@ -22,7 +22,7 @@ #include // constructor -FieldReduction::FieldReduction (std::string rd_name) +FieldReduction::FieldReduction (const std::string& rd_name) : ReducedDiags{rd_name} { using namespace amrex::literals; diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H index 0ecb74d4bf5..cfba804d0f0 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H @@ -54,7 +54,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - LoadBalanceCosts(std::string rd_name); + LoadBalanceCosts(const std::string& rd_name); /** * This function updates the costs, given the current distribution mapping, diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index b4e07b51982..882c94b12eb 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -7,6 +7,7 @@ #include "LoadBalanceCosts.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -35,6 +36,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { @@ -65,7 +67,7 @@ namespace } // constructor -LoadBalanceCosts::LoadBalanceCosts (std::string rd_name) +LoadBalanceCosts::LoadBalanceCosts (const std::string& rd_name) : ReducedDiags{rd_name} { } @@ -125,7 +127,7 @@ void LoadBalanceCosts::ComputeDiags (int step) for (int lev = 0; lev < nLevels; ++lev) { const amrex::DistributionMapping& dm = warpx.DistributionMap(lev); - const MultiFab & Ex = warpx.getEfield(lev,0); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); for (MFIter mfi(Ex, false); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H index a43868ccc5f..c58f78deedb 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - LoadBalanceEfficiency(std::string rd_name); + LoadBalanceEfficiency(const std::string& rd_name); /** * This function gets the current load balance efficiency diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp index 6dd066d5324..b6f3a58e670 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp @@ -20,7 +20,7 @@ using namespace amrex; // constructor -LoadBalanceEfficiency::LoadBalanceEfficiency (std::string rd_name) +LoadBalanceEfficiency::LoadBalanceEfficiency (const std::string& rd_name) : ReducedDiags{rd_name} { // read number of levels diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H index b0df66709f3..cb3fb8c4f08 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H @@ -25,7 +25,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleEnergy(std::string rd_name); + ParticleEnergy(const std::string& rd_name); /** * This function computes the particle relativistic kinetic energy (EP). diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index 157b4bfb368..9446af5e20c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -34,7 +34,7 @@ using namespace amrex; // constructor -ParticleEnergy::ParticleEnergy (std::string rd_name) +ParticleEnergy::ParticleEnergy (const std::string& rd_name) : ReducedDiags{rd_name} { // get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H index ed5cce1d7e0..d3505b5ccf7 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H @@ -25,7 +25,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleExtrema(std::string rd_name); + ParticleExtrema(const std::string& rd_name); /// name of species std::string m_species_name; diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index ba8e7e373de..0cc5429be7a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -11,6 +11,7 @@ #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif +#include "FieldSolver/Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" #include "Particles/MultiParticleContainer.H" @@ -51,9 +52,10 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor -ParticleExtrema::ParticleExtrema (std::string rd_name) +ParticleExtrema::ParticleExtrema (const std::string& rd_name) : ReducedDiags{rd_name} { // read species name @@ -373,12 +375,12 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); const GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H index 0b832dedf14..4145a58fb95 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H @@ -28,7 +28,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleHistogram(std::string rd_name); + ParticleHistogram(const std::string& rd_name); /// normalization type int m_norm; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index 7acde30761c..5343de2ba68 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -50,8 +50,8 @@ struct NormalizationType { }; // constructor -ParticleHistogram::ParticleHistogram (std::string rd_name): - ReducedDiags{rd_name} +ParticleHistogram::ParticleHistogram (const std::string& rd_name) +: ReducedDiags{rd_name} { const ParmParse pp_rd_name(rd_name); diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H index 6ba85502819..0c648528920 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H @@ -30,7 +30,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleHistogram2D(std::string rd_name); + ParticleHistogram2D(const std::string& rd_name); /// File type std::string m_openpmd_backend {"default"}; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index bcc22eac598..8dd19186b25 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -54,7 +54,7 @@ using namespace amrex; // constructor -ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) +ParticleHistogram2D::ParticleHistogram2D (const std::string& rd_name) : ReducedDiags{rd_name} { ParmParse pp_rd_name(rd_name); @@ -138,7 +138,7 @@ void ParticleHistogram2D::ComputeDiags (int step) Array tlo{0,0}; // lower bounds Array thi{m_bin_num_abs-1, m_bin_num_ord-1}; // inclusive upper bounds amrex::TableData d_data_2D(tlo, thi); - m_h_data_2D = amrex::TableData (tlo, thi, The_Pinned_Arena()); + m_h_data_2D.resize(tlo, thi, The_Pinned_Arena()); auto const& h_table_data = m_h_data_2D.table(); // Initialize data on the host @@ -316,6 +316,8 @@ void ParticleHistogram2D::WriteToFile (int step) const {static_cast(m_bin_num_ord), static_cast(m_bin_num_abs)}); series.flush(); + i.close(); + series.close(); #else amrex::ignore_unused(step); WARPX_ABORT_WITH_MESSAGE("ParticleHistogram2D: Needs openPMD-api compiled into WarpX, but was not found!"); diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H index feb8957e516..ea079f92e62 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H @@ -25,7 +25,7 @@ public: * * \param[in] rd_name reduced diags names */ - ParticleMomentum(std::string rd_name); + ParticleMomentum(const std::string& rd_name); /** * \brief This function computes the particle relativistic momentum, diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5f278d48f14..47d1ec2cdd4 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -32,7 +32,7 @@ using namespace amrex; -ParticleMomentum::ParticleMomentum (std::string rd_name) +ParticleMomentum::ParticleMomentum (const std::string& rd_name) : ReducedDiags{rd_name} { // Get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.H b/Source/Diagnostics/ReducedDiags/ParticleNumber.H index 3143a387a74..894048806ca 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.H +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleNumber(std::string rd_name); + ParticleNumber(const std::string& rd_name); /** * This function computes the total number of macroparticles and physical particles of each diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp index 8c0377a3a82..bfdf1daffe3 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp @@ -27,7 +27,7 @@ using namespace amrex::literals; // constructor -ParticleNumber::ParticleNumber (std::string rd_name) +ParticleNumber::ParticleNumber (const std::string& rd_name) : ReducedDiags{rd_name} { // get a reference to WarpX instance @@ -112,27 +112,13 @@ void ParticleNumber::ComputeDiags (int step) for (int i_s = 0; i_s < nSpecies; ++i_s) { // get WarpXParticleContainer class object - const auto & myspc = mypc.GetParticleContainer(i_s); + auto & myspc = mypc.GetParticleContainer(i_s); // Save total number of macroparticles for this species m_data[idx_first_species_macroparticles + i_s] = myspc.TotalNumberOfParticles(); - using PType = typename WarpXParticleContainer::SuperParticleType; - - // Reduction to compute sum of weights for this species - auto Wtot = ReduceSum( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> amrex::Real - { - return p.rdata(PIdx::w); - }); - - // MPI reduction - amrex::ParallelDescriptor::ReduceRealSum - (Wtot, amrex::ParallelDescriptor::IOProcessorNumber()); - // Save sum of particles weight for this species - m_data[idx_first_species_sum_weight + i_s] = Wtot; - + m_data[idx_first_species_sum_weight + i_s] = myspc.sumParticleWeight(false); // Increase total number of macroparticles and total weight (all species) m_data[idx_total_macroparticles] += m_data[idx_first_species_macroparticles + i_s]; diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 09aa85586be..2c942e1df6d 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -52,7 +52,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ReducedDiags (std::string rd_name); + ReducedDiags (const std::string& rd_name); /** * Virtual destructor for polymorphism diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 483f4c68ece..e3835dde416 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -23,8 +23,8 @@ using namespace amrex; // constructor -ReducedDiags::ReducedDiags (std::string rd_name) - : m_rd_name(std::move(rd_name)) +ReducedDiags::ReducedDiags (const std::string& rd_name): +m_rd_name{rd_name} { BackwardCompatibility(); diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.H b/Source/Diagnostics/ReducedDiags/RhoMaximum.H index 6f306a5e7ad..ceff0f666e0 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.H +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.H @@ -28,7 +28,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - RhoMaximum(std::string rd_name); + RhoMaximum(const std::string& rd_name); /** * This function computes the maximum and minimum values of rho (summed over all species) and diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 4613a70b3b0..9902415d9a9 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -29,7 +29,7 @@ using namespace amrex::literals; // constructor -RhoMaximum::RhoMaximum (std::string rd_name) +RhoMaximum::RhoMaximum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/SliceDiagnostic.H b/Source/Diagnostics/SliceDiagnostic.H index 000b01d470f..570f86d5384 100644 --- a/Source/Diagnostics/SliceDiagnostic.H +++ b/Source/Diagnostics/SliceDiagnostic.H @@ -25,10 +25,10 @@ void CheckSliceInput( amrex::RealBox real_box, amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); void InterpolateSliceValues( amrex::MultiFab& smf, - amrex::IntVect interp_lo, amrex::RealBox slice_realbox, - amrex::Vector geom, int ncomp, int nghost, - amrex::IntVect slice_lo, amrex::IntVect slice_hi, - amrex::IntVect SliceType, amrex::RealBox real_box); + amrex::IntVect interp_lo, amrex::RealBox slice_realbox, + const amrex::Vector& geom, int ncomp, int nghost, + amrex::IntVect slice_lo, amrex::IntVect slice_hi, + amrex::IntVect SliceType, amrex::RealBox real_box); void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, amrex::IntVect slice_lo, amrex::Vector geom, diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 71a585aea35..14a2d633a0c 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -7,8 +7,9 @@ */ #include "SliceDiagnostic.H" -#include "WarpX.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" +#include "WarpX.H" #include #include @@ -40,7 +41,7 @@ #include using namespace amrex; - +using namespace warpx::fields; /* \brief * The functions creates the slice for diagnostics based on the user-input. @@ -195,27 +196,27 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio); } - if( SliceType == WarpX::GetInstance().getEfield(0,0).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,0).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getEfield(0,1).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,1).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getEfield(0,2).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,2).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } - if( SliceType == WarpX::GetInstance().getBfield(0,0).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,0).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getBfield(0,1).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,1).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getBfield(0,2).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,2).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } @@ -419,7 +420,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, */ void InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, - Vector geom, int ncomp, int nghost, + const Vector& geom, int ncomp, int nghost, IntVect slice_lo, IntVect /*slice_hi*/, IntVect SliceType, const RealBox real_box) { diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 6c904790e15..99d6e0682ab 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -91,12 +91,12 @@ public: * @param authors a string specifying the authors of the simulation (can be empty) */ WarpXOpenPMDPlot (openPMD::IterationEncoding ie, - std::string filetype, - std::string operator_type, - std::map< std::string, std::string > operator_parameters, - std::string engine_type, - std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections, + const std::string& filetype, + const std::string& operator_type, + const std::map< std::string, std::string >& operator_parameters, + const std::string& engine_type, + const std::map< std::string, std::string >& engine_parameters, + const std::vector& fieldPMLdirections, const std::string& authors); ~WarpXOpenPMDPlot (); @@ -188,9 +188,9 @@ private: void SetupMeshComp ( openPMD::Mesh& mesh, - amrex::Geometry& full_geom, - std::string comp_name, - std::string field_name, + amrex::Geometry const& full_geom, + std::string const& comp_name, + std::string const& field_name, amrex::MultiFab const& mf, bool var_in_theta_mode ) const; @@ -218,11 +218,13 @@ private: /** This function sets up the entries for storing the particle positions and global IDs * * @param[in] currSpecies Corresponding openPMD species + * @param[in] positionComponents user-selected components of the particle position * @param[in] np Number of particles * @param[in] isBTD Is this a back-transformed diagnostics output? */ void SetupPos ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, bool isBTD = false); @@ -231,12 +233,14 @@ private: * Sets the entries for storing particle position offset, constant records (charge, mass) and ED-PIC attributes. * * @param[in] currSpecies Corresponding openPMD species + * @param[in] positionComponents user-selected components of the particle position * @param[in] np Number of particles * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles */ void SetConstParticleRecordsEDPIC ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, amrex::ParticleReal charge, amrex::ParticleReal mass); @@ -284,8 +288,8 @@ private: * @param[in] name species name * @param[in] iteration timestep * @param[in] write_real_comp The real attribute ids, from WarpX - * @param[in] real_comp_names The real attribute names, from WarpX * @param[in] write_int_comp The int attribute ids, from WarpX + * @param[in] real_comp_names The real attribute names, from WarpX * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 491cdccff8e..5afb49db597 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -213,32 +213,29 @@ namespace detail return make_pair(record_name, component_name); } - /** Return the component labels for particle positions + /** Return the user-selected components for particle positions + * + * @param[in] write_real_comp The real attribute ids, from WarpX + * @param[in] real_comp_names The real attribute names, from WarpX */ inline std::vector< std::string > - getParticlePositionComponentLabels (bool ignore_dims=false) + getParticlePositionComponentLabels ( + amrex::Vector const & write_real_comp, + amrex::Vector const & real_comp_names) { - using vs = std::vector< std::string >; - auto positionComponents = vs{"x", "y", "z"}; - if (!ignore_dims) { -#if defined(WARPX_DIM_1D_Z) - positionComponents = vs{"z"}; -#elif defined(WARPX_DIM_XZ) - positionComponents = vs{"x", "z"}; -#elif defined(WARPX_DIM_RZ) - // note: although we internally store particle positions - // for AMReX in r,z and a theta attribute, we - // actually need them for algorithms (e.g. push) - // and I/O in Cartesian. - // Other attributes like momentum are consequently - // stored in x,y,z internally. - positionComponents = vs{"x", "y", "z"}; -#elif defined(WARPX_DIM_3D) - positionComponents = vs{"x", "y", "z"}; -#else -# error Unknown WarpX dimensionality. -#endif + std::vector< std::string > positionComponents; + + int idx = 0; + for (auto const & comp : real_comp_names ) { + if (write_real_comp[idx]) { + if (comp == "position_x" || comp == "position_y" || comp == "position_z") { + std::string const last_letter{comp.back()}; + positionComponents.push_back(last_letter); + } + } + idx++; } + return positionComponents; } @@ -343,7 +340,7 @@ namespace detail * set the metadata that indicates the physical unit. */ inline void - setOpenPMDUnit ( openPMD::Mesh mesh, const std::string field_name ) + setOpenPMDUnit ( openPMD::Mesh mesh, const std::string& field_name ) { if (field_name[0] == 'E'){ // Electric field mesh.setUnitDimension({ @@ -377,19 +374,19 @@ namespace detail #ifdef WARPX_USE_OPENPMD WarpXOpenPMDPlot::WarpXOpenPMDPlot ( openPMD::IterationEncoding ie, - std::string openPMDFileType, - std::string operator_type, - std::map< std::string, std::string > operator_parameters, - std::string engine_type, - std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections, + const std::string& openPMDFileType, + const std::string& operator_type, + const std::map< std::string, std::string >& operator_parameters, + const std::string& engine_type, + const std::map< std::string, std::string >& engine_parameters, + const std::vector& fieldPMLdirections, const std::string& authors) : m_Series(nullptr), m_MPIRank{amrex::ParallelDescriptor::MyProc()}, m_MPISize{amrex::ParallelDescriptor::NProcs()}, m_Encoding(ie), - m_OpenPMDFileType(std::move(openPMDFileType)), - m_fieldPMLdirections(std::move(fieldPMLdirections)), + m_OpenPMDFileType{openPMDFileType}, + m_fieldPMLdirections{fieldPMLdirections}, m_authors{authors} { m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters, @@ -548,48 +545,6 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { pinned_pc->make_alike() : pc->make_alike(); - // names of amrex::Real and int particle attributes in SoA data - amrex::Vector real_names; - amrex::Vector int_names; - amrex::Vector int_flags; - amrex::Vector real_flags; - - // see openPMD ED-PIC extension for namings - // note: an underscore separates the record name from its component - // for non-scalar records - // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX -#if !defined (WARPX_DIM_1D_Z) - real_names.push_back("position_x"); -#endif -#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - real_names.push_back("position_y"); -#endif - real_names.push_back("position_z"); - real_names.push_back("weighting"); - real_names.push_back("momentum_x"); - real_names.push_back("momentum_y"); - real_names.push_back("momentum_z"); - - // get the names of the real comps - real_names.resize(tmp.NumRealComps()); - auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) - { - real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); - } - // plot any "extra" fields by default - real_flags = particle_diags[i].m_plot_flags; - real_flags.resize(tmp.NumRealComps(), 1); - // and the names - int_names.resize(tmp.NumIntComps()); - auto runtime_inames = tmp.getParticleRuntimeiComps(); - for (auto const& x : runtime_inames) - { - int_names[x.second+0] = detail::snakeToCamel(x.first); - } - // plot by default - int_flags.resize(tmp.NumIntComps(), 1); - auto rtmap = pc->getParticleComps(); const auto mass = pc->AmIA() ? PhysConst::m_e : pc->getMass(); @@ -633,6 +588,51 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); } + // Gather the electrostatic potential (phi) on the macroparticles + if ( particle_diags[i].m_plot_phi ) { + storePhiOnParticles( tmp, WarpX::electrostatic_solver_id, !use_pinned_pc ); + } + + // names of amrex::Real and int particle attributes in SoA data + amrex::Vector real_names; + amrex::Vector int_names; + amrex::Vector int_flags; + amrex::Vector real_flags; + // see openPMD ED-PIC extension for namings + // note: an underscore separates the record name from its component + // for non-scalar records + // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX +#if !defined (WARPX_DIM_1D_Z) + real_names.push_back("position_x"); +#endif +#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + real_names.push_back("position_y"); +#endif + real_names.push_back("position_z"); + real_names.push_back("weighting"); + real_names.push_back("momentum_x"); + real_names.push_back("momentum_y"); + real_names.push_back("momentum_z"); + // get the names of the real comps + real_names.resize(tmp.NumRealComps()); + auto runtime_rnames = tmp.getParticleRuntimeComps(); + for (auto const& x : runtime_rnames) + { + real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); + } + // plot any "extra" fields by default + real_flags = particle_diags[i].m_plot_flags; + real_flags.resize(tmp.NumRealComps(), 1); + // and the names + int_names.resize(tmp.NumIntComps()); + auto runtime_inames = tmp.getParticleRuntimeiComps(); + for (auto const& x : runtime_inames) + { + int_names[x.second+0] = detail::snakeToCamel(x.first); + } + // plot by default + int_flags.resize(tmp.NumIntComps(), 1); + // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. DumpToFile(&tmp, @@ -642,9 +642,8 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { int_flags, real_names, int_names, pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush - ); -} + isBTD, isLastBTDFlush); + } } void @@ -705,16 +704,18 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; } + auto const positionComponents = detail::getParticlePositionComponentLabels(write_real_comp, real_comp_names); + // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 if (doParticleSetup || is_resizing_flush) { - SetupPos(currSpecies, NewParticleVectorSize, isBTD); + SetupPos(currSpecies, positionComponents, NewParticleVectorSize, isBTD); SetupRealProperties(pc, currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, NewParticleVectorSize, isBTD); } if (is_last_flush_to_step) { - SetConstParticleRecordsEDPIC(currSpecies, NewParticleVectorSize, charge, mass); + SetConstParticleRecordsEDPIC(currSpecies, positionComponents, NewParticleVectorSize, charge, mass); } // open files from all processors, in case some will not contribute below @@ -818,7 +819,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, // the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names // so using the min to be safe. // - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + auto const getComponentRecord = [&currSpecies](std::string const& comp_name) { // handle scalar and non-scalar records by name const auto [record_name, component_name] = detail::name2openPMD(comp_name); return currSpecies[record_name][component_name]; @@ -895,7 +896,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, auto const numParticleOnTile64 = static_cast(numParticleOnTile); auto const& soa = pti.GetStructOfArrays(); - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + auto const getComponentRecord = [&currSpecies](std::string const& comp_name) { // handle scalar and non-scalar records by name const auto [record_name, component_name] = detail::name2openPMD(comp_name); return currSpecies[record_name][component_name]; @@ -978,6 +979,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, void WarpXOpenPMDPlot::SetupPos ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, bool const isBTD) { @@ -986,7 +988,6 @@ WarpXOpenPMDPlot::SetupPos ( auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); - auto const positionComponents = detail::getParticlePositionComponentLabels(); for( auto const& comp : positionComponents ) { currSpecies["position"][comp].resetDataset( realType ); } @@ -998,6 +999,7 @@ WarpXOpenPMDPlot::SetupPos ( void WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, amrex::ParticleReal const charge, amrex::ParticleReal const mass) @@ -1006,7 +1008,6 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles - auto const positionComponents = detail::getParticlePositionComponentLabels(true); for( auto const& comp : positionComponents ) { currSpecies["positionOffset"][comp].resetDataset( realType ); } @@ -1015,9 +1016,12 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( #if defined(WARPX_DIM_1D_Z) currSpecies["position"]["x"].resetDataset( realType ); currSpecies["position"]["y"].resetDataset( realType ); + currSpecies["positionOffset"]["x"].resetDataset( realType ); + currSpecies["positionOffset"]["y"].resetDataset( realType ); #endif #if defined(WARPX_DIM_XZ) currSpecies["position"]["y"].resetDataset( realType ); + currSpecies["positionOffset"]["y"].resetDataset( realType ); #endif // make constant @@ -1031,22 +1035,29 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( #if defined(WARPX_DIM_1D_Z) currSpecies["position"]["x"].makeConstant( 0._prt ); currSpecies["position"]["y"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["x"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["y"].makeConstant( 0._prt ); #endif #if defined(WARPX_DIM_XZ) currSpecies["position"]["y"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["y"].makeConstant( 0._prt ); #endif // meta data - currSpecies["position"].setUnitDimension( detail::getUnitDimension("position") ); - currSpecies["positionOffset"].setUnitDimension( detail::getUnitDimension("positionOffset") ); + if (!positionComponents.empty()) { + currSpecies["position"].setUnitDimension( detail::getUnitDimension("position") ); + currSpecies["positionOffset"].setUnitDimension( detail::getUnitDimension("positionOffset") ); + } currSpecies["charge"].setUnitDimension( detail::getUnitDimension("charge") ); currSpecies["mass"].setUnitDimension( detail::getUnitDimension("mass") ); // meta data for ED-PIC extension - currSpecies["position"].setAttribute( "macroWeighted", 0u ); - currSpecies["position"].setAttribute( "weightingPower", 0.0 ); - currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); - currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + if (!positionComponents.empty()) { + currSpecies["position"].setAttribute( "macroWeighted", 0u ); + currSpecies["position"].setAttribute( "weightingPower", 0.0 ); + currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); + currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + } currSpecies["id"].setAttribute( "macroWeighted", 0u ); currSpecies["id"].setAttribute( "weightingPower", 0.0 ); currSpecies["charge"].setAttribute( "macroWeighted", 0u ); @@ -1196,9 +1207,9 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, */ void WarpXOpenPMDPlot::SetupMeshComp (openPMD::Mesh& mesh, - amrex::Geometry& full_geom, - std::string comp_name, - std::string field_name, + amrex::Geometry const& full_geom, + std::string const& comp_name, + std::string const& field_name, amrex::MultiFab const& mf, bool var_in_theta_mode) const { diff --git a/Source/EmbeddedBoundary/DistanceToEB.H b/Source/EmbeddedBoundary/DistanceToEB.H index b46def938db..7ee47c1172c 100644 --- a/Source/EmbeddedBoundary/DistanceToEB.H +++ b/Source/EmbeddedBoundary/DistanceToEB.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef DISTANCETOEB_H_ -#define DISTANCETOEB_H_ +#ifndef WARPX_DISTANCETOEB_H_ +#define WARPX_DISTANCETOEB_H_ #include "Utils/TextMsg.H" @@ -129,4 +129,4 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP } #endif // AMREX_USE_EB -#endif // DISTANCETOEB_H_ +#endif // WARPX_DISTANCETOEB_H_ diff --git a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H index 3099f5df43d..345cafe7b4f 100644 --- a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H +++ b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARYPROCESS_H_ -#define PARTICLEBOUNDARYPROCESS_H_ +#ifndef WARPX_PARTICLEBOUNDARYPROCESS_H_ +#define WARPX_PARTICLEBOUNDARYPROCESS_H_ #include #include @@ -36,4 +36,4 @@ struct Absorb { }; } -#endif +#endif //WARPX_PARTICLEBOUNDARYPROCESS_H_ diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 7d6934cbb9c..1e915b39381 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLESCRAPER_H_ -#define PARTICLESCRAPER_H_ +#ifndef WARPX_PARTICLESCRAPER_H_ +#define WARPX_PARTICLESCRAPER_H_ #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -65,9 +65,9 @@ */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) { - scrapeParticles(pc, distance_to_eb, lev, lev, std::forward(f)); + scrapeParticlesAtEB(pc, distance_to_eb, lev, lev, std::forward(f)); } /** @@ -108,9 +108,9 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, F&& f) +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, F&& f) { - scrapeParticles(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); + scrapeParticlesAtEB(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); } /** @@ -153,10 +153,10 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev_min, int lev_max, F&& f) { - BL_PROFILE("scrapeParticles"); + BL_PROFILE("scrapeParticlesAtEB"); for (int lev = lev_min; lev <= lev_max; ++lev) { @@ -183,15 +183,16 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { int ic, jc, kc; // Cell-centered indices - int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, ic, jc, kc, Wc); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); DistanceToEB::normalize(normal); amrex::RealVect pos; @@ -216,4 +217,4 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t } } -#endif +#endif //WARPX_PARTICLESCRAPER_H_ diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index c215311cd47..392302e59de 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -70,8 +70,8 @@ WarpX::Evolve (int numsteps) // Note that the default argument is numsteps = -1 const int numsteps_max = (numsteps < 0)?(max_step):(istep[0] + numsteps); - bool early_params_checked = false; // check typos in inputs after step 1 - bool exit_loop_due_to_interrupt_signal = false; + // check typos in inputs after step 1 + bool early_params_checked = false; static Real evolve_time = 0; @@ -81,7 +81,7 @@ WarpX::Evolve (int numsteps) WARPX_PROFILE("WarpX::Evolve::step"); const auto evolve_time_beg_step = static_cast(amrex::second()); - //Check and clear signal flags and asynchronously broadcast them from process 0 + // Check and clear signal flags and asynchronously broadcast them from process 0 SignalHandling::CheckSignals(); multi_diags->NewIteration(); @@ -92,90 +92,29 @@ WarpX::Evolve (int numsteps) } ExecutePythonCallback("beforestep"); - amrex::LayoutData* cost = WarpX::getCosts(0); - if (cost) { - if (step > 0 && load_balance_intervals.contains(step+1)) - { - LoadBalance(); + CheckLoadBalance(step); - // Reset the costs to 0 - ResetCosts(); - } - for (int lev = 0; lev <= finest_level; ++lev) - { - cost = WarpX::getCosts(lev); - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - // Perform running average of the costs - // (Giving more importance to most recent costs; only needed - // for timers update, heuristic load balance considers the - // instantaneous costs) - for (const auto& i : cost->IndexArray()) - { - (*cost)[i] *= (1._rt - 2._rt/load_balance_intervals.localPeriod(step+1)); - } - } - } - } - - if (evolve_scheme == EvolveScheme::Explicit) { - // At the beginning, we have B^{n} and E^{n}. - // Particles have p^{n} and x^{n}. - // is_synchronized is true. - - if (is_synchronized) { - // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ng_alloc_EB); - FillBoundaryB(guard_cells.ng_alloc_EB); - - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - // on first step, push p by -0.5*dt - for (int lev = 0; lev <= finest_level; ++lev) - { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - - } else { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); - } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - FillBoundaryAux(guard_cells.ng_UpdateAux); - } - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - } + if (evolve_scheme == EvolveScheme::Explicit) + { + ExplicitFillBoundaryEBUpdateAux(); } // If needed, deposit the initial ion charge and current densities that // will be used to update the E-field in Ohm's law. if (step == step_begin && electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC - ) { HybridPICDepositInitialRhoAndJ(); } + ) { + HybridPICDepositInitialRhoAndJ(); + } // Run multi-physics modules: // ionization, Coulomb collisions, QED doFieldIonization(); + ExecutePythonCallback("beforecollisions"); mypc->doCollisions( cur_time, dt[0] ); ExecutePythonCallback("aftercollisions"); + #ifdef WARPX_QED doQEDEvents(); mypc->doQEDSchwinger(); @@ -186,6 +125,7 @@ WarpX::Evolve (int numsteps) ExecutePythonCallback("particleinjection"); + // TODO: move out if (evolve_scheme == EvolveScheme::ImplicitPicard || evolve_scheme == EvolveScheme::SemiImplicitPicard) { OneStep_ImplicitPicard(cur_time); @@ -229,12 +169,13 @@ WarpX::Evolve (int numsteps) mypc->doResampling(istep[0]+1, verbose); if (evolve_scheme == EvolveScheme::Explicit) { - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date + } + // TODO: move out + if (evolve_scheme == EvolveScheme::Explicit) { if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { // At the end of last step, push p by 0.5*dt to synchronize FillBoundaryE(guard_cells.ng_FieldGather); @@ -284,49 +225,7 @@ WarpX::Evolve (int numsteps) } } - mypc->ContinuousFluxInjection(cur_time, dt[0]); - - mypc->ApplyBoundaryConditions(); - - // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); -#endif - - m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); - - // Non-Maxwell solver: particles can move by an arbitrary number of cells - if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || - electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) - { - mypc->Redistribute(); - } - else - { - // Electromagnetic solver: due to CFL condition, particles can - // only move by one or two cells per time step - if (max_level == 0) { - int num_redistribute_ghost = num_moved; - if ((m_v_galilean[0]!=0) or (m_v_galilean[1]!=0) or (m_v_galilean[2]!=0)) { - // Galilean algorithm ; particles can move by up to 2 cells - num_redistribute_ghost += 2; - } else { - // Standard algorithm ; particles can move by up to 1 cell - num_redistribute_ghost += 1; - } - mypc->RedistributeLocal(num_redistribute_ghost); - } - else { - mypc->Redistribute(); - } - } - - if (sort_intervals.contains(step+1)) { - if (verbose) { - amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); - } - mypc->SortParticlesByBin(sort_bin_size); - } + HandleParticlesAtBoundaries(step, cur_time, num_moved); // Field solve step for electrostatic or hybrid-PIC solvers if( electrostatic_solver_id != ElectrostaticSolverAlgo::None || @@ -378,12 +277,7 @@ WarpX::Evolve (int numsteps) // inputs: unused parameters (e.g. typos) check after step 1 has finished if (!early_params_checked) { - amrex::Print() << "\n"; // better: conditional \n based on return value - amrex::ParmParse::QueryUnusedInputs(); - - //Print the warning list right after the first step. - amrex::Print() << - ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); + checkEarlyUnusedParams(); early_params_checked = true; } @@ -401,20 +295,18 @@ WarpX::Evolve (int numsteps) << " s; Avg. per step = " << evolve_time/(step-step_begin+1) << " s\n\n"; } - exit_loop_due_to_interrupt_signal = SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK); - if (cur_time >= stop_time - 1.e-3*dt[0] || exit_loop_due_to_interrupt_signal) { + if (checkStopSimulation(cur_time)) { break; } + } // End loop on time steps - // End loop on time steps - } // This if statement is needed for PICMI, which allows the Evolve routine to be // called multiple times, otherwise diagnostics will be done at every call, // regardless of the diagnostic period parameter provided in the inputs. if (istep[0] == max_step || (stop_time - 1.e-3*dt[0] <= cur_time && cur_time < stop_time + dt[0]) - || exit_loop_due_to_interrupt_signal) { + || m_exit_loop_due_to_interrupt_signal) { multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } + if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -526,6 +418,120 @@ WarpX::OneStep_nosub (Real cur_time) ExecutePythonCallback("afterEsolve"); } +bool WarpX::checkStopSimulation (amrex::Real cur_time) +{ + m_exit_loop_due_to_interrupt_signal = SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK); + return (cur_time >= stop_time - 1.e-3*dt[0]) || + m_exit_loop_due_to_interrupt_signal; +} + +void WarpX::checkEarlyUnusedParams () +{ + amrex::Print() << "\n"; // better: conditional \n based on return value + amrex::ParmParse::QueryUnusedInputs(); + + // Print the warning list right after the first step. + amrex::Print() << ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); +} + +void WarpX::ExplicitFillBoundaryEBUpdateAux () +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(evolve_scheme == EvolveScheme::Explicit, + "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux wihtout Explicit evolve scheme set!"); + + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + + if (is_synchronized) { + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) + { + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + } + is_synchronized = false; + + } else { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + } +} + +void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved) +{ + mypc->ContinuousFluxInjection(cur_time, dt[0]); + + mypc->ApplyBoundaryConditions(); + m_particle_boundary_buffer->gatherParticlesFromDomainBoundaries(*mypc); + + // Non-Maxwell solver: particles can move by an arbitrary number of cells + if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) + { + mypc->Redistribute(); + } + else + { + // Electromagnetic solver: due to CFL condition, particles can + // only move by one or two cells per time step + if (max_level == 0) { + int num_redistribute_ghost = num_moved; + if ((m_v_galilean[0]!=0) or (m_v_galilean[1]!=0) or (m_v_galilean[2]!=0)) { + // Galilean algorithm ; particles can move by up to 2 cells + num_redistribute_ghost += 2; + } else { + // Standard algorithm ; particles can move by up to 1 cell + num_redistribute_ghost += 1; + } + mypc->RedistributeLocal(num_redistribute_ghost); + } + else { + mypc->Redistribute(); + } + } + + // interact the particles with EB walls (if present) +#ifdef AMREX_USE_EB + mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); + m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + mypc->deleteInvalidParticles(); +#endif + + if (sort_intervals.contains(step+1)) { + if (verbose) { + amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); + } + mypc->SortParticlesByBin(sort_bin_size); + } +} + void WarpX::SyncCurrentAndRho () { if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -639,7 +645,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (dt[0] denotes the time step on mesh refinement level 0) if (J_in_time == JInTime::Linear) { - auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; + auto& current = (do_current_centering) ? current_fp_nodal : current_fp; mypc->DepositCurrent(current, dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', @@ -673,7 +679,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) - auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; + auto& current = (do_current_centering) ? current_fp_nodal : current_fp; mypc->DepositCurrent(current, dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', @@ -1061,8 +1067,13 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, * The mirror normal direction has to be parallel to the z axis. */ void -WarpX::applyMirrors(Real time) +WarpX::applyMirrors (Real time) { + // something to do? + if (num_mirrors == 0) { + return; + } + // Loop over the mirrors for(int i_mirror=0; i_mirror #include @@ -56,7 +56,7 @@ namespace ElectrostaticSolver { * * \param [in] potential The string value of the potential */ - void setPotentialEB (std::string potential) { + void setPotentialEB(const std::string& potential) { potential_eb_str = potential; buildParsersEB(); } @@ -122,4 +122,4 @@ namespace ElectrostaticSolver { }; } // namespace ElectrostaticSolver -#endif // ELECTROSTATICSOLVER_H_ +#endif // WARPX_ELECTROSTATICSOLVER_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index bd3086c0438..a74beae3c91 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -55,6 +55,7 @@ #include using namespace amrex; +using namespace warpx::fields; void WarpX::ComputeSpaceChargeField (bool const reset_fields) @@ -88,7 +89,8 @@ WarpX::ComputeSpaceChargeField (bool const reset_fields) } // Add the field due to the boundary potentials - if (electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic){ + if (m_boundary_potential_specified || + (electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic)){ AddBoundaryField(); } } @@ -145,6 +147,10 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) { WARPX_PROFILE("WarpX::AddSpaceChargeField"); + if (pc.getCharge() == 0) { + return; + } + // Store the boundary conditions for the field solver if they haven't been // stored yet if (!m_poisson_boundary_handler.bcs_set) { @@ -159,6 +165,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Allocate fields for charge and potential const int num_levels = max_level + 1; Vector > rho(num_levels); + Vector > rho_coarse(num_levels); // Used in order to interpolate between levels Vector > phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; @@ -169,15 +176,33 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) rho[lev]->setVal(0.); phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); phi[lev]->setVal(0.); + if (lev > 0) { + // For MR levels: allocated the coarsened version of rho + BoxArray cba = nba; + cba.coarsen(refRatio(lev-1)); + rho_coarse[lev] = std::make_unique(cba, DistributionMap(lev), 1, ng); + rho_coarse[lev]->setVal(0.); + } } // Deposit particle charge density (source of Poisson solver) - bool const local = false; + // The options below are identical to those in MultiParticleContainer::DepositCharge + bool const local = true; bool const reset = false; bool const apply_boundary_and_scale_volume = true; + bool const interpolate_across_levels = false; if ( !pc.do_not_deposit) { - pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume); + pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, + interpolate_across_levels); + } + for (int lev = 0; lev <= max_level; lev++) { + if (lev > 0) { + if (charge_buf[lev]) { + charge_buf[lev]->setVal(0.); + } + } } + SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels // Get the particle beta vector bool const local_average = false; // Average across all MPI ranks @@ -220,7 +245,13 @@ WarpX::AddSpaceChargeFieldLabFrame () int const lev = 0; myfl->DepositCharge( lev, *rho_fp[lev] ); } - + for (int lev = 0; lev <= max_level; lev++) { + if (lev > 0) { + if (charge_buf[lev]) { + charge_buf[lev]->setVal(0.); + } + } + } SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels #ifndef WARPX_DIM_RZ for (int lev = 0; lev <= finestLevel(); lev++) { @@ -248,7 +279,7 @@ WarpX::AddSpaceChargeFieldLabFrame () // Use the tridiag solver with 1D computePhiTriDiagonal(rho_fp, phi_fp); #else - // Use the AMREX MLMG solver otherwise + // Use the AMREX MLMG or the FFT (IGF) solver otherwise computePhi(rho_fp, phi_fp, beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity); @@ -319,18 +350,18 @@ WarpX::computePhi (const amrex::Vector >& rho, e_field.push_back( # if defined(WARPX_DIM_1D_Z) amrex::Array{ - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 2) } # elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array{ - get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 2) } # elif defined(WARPX_DIM_3D) amrex::Array{ - get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2) } # endif ); @@ -351,6 +382,9 @@ WarpX::computePhi (const amrex::Vector >& rho, const std::optional > eb_farray_box_factory; #endif + bool const is_solver_multigrid = + WarpX::poisson_solver_id != PoissonSolverAlgo::IntegratedGreenFunction; + ablastr::fields::computePhi( sorted_rho, sorted_phi, @@ -363,6 +397,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->dmap, this->grids, this->m_poisson_boundary_handler, + is_solver_multigrid, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, @@ -1000,6 +1035,7 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo amrex::ignore_unused(geom); #endif for (int idim=dim_start; idim, 3 >& Bfield, amrex::Box domain_box, amrex::Real const dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi) { + amrex::Vector field_boundary_lo, + amrex::Vector field_boundary_hi) { // Ensure that we are using the Yee solver WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H index b762530e1a3..c4978287aec 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H @@ -100,6 +100,25 @@ struct CartesianYeeAlgorithm { #endif } + /** + * Perform second derivative along x on a cell-centered grid, from a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dxx ( + T_Field const& F, + amrex::Real const * const coefs_x, int const /*n_coefs_x*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; +#if (defined WARPX_DIM_1D_Z) + amrex::ignore_unused(F, coefs_x, i, j, k, ncomp); + return 0._rt; // 1D Cartesian: derivative along x is 0 +#else + amrex::Real const inv_dx2 = coefs_x[0]*coefs_x[0]; + return inv_dx2*( F(i-1,j,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i+1,j,k,ncomp) ); +#endif + } + /** * Perform derivative along y on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -147,6 +166,25 @@ struct CartesianYeeAlgorithm { #endif } + /** + * Perform derivative along y on a nodal grid, from a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dyy ( + T_Field const& F, + amrex::Real const * const coefs_y, int const /*n_coefs_y*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; +#if defined WARPX_DIM_3D + Real const inv_dy2 = coefs_y[0]*coefs_y[0]; + return inv_dy2*( F(i,j-1,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j+1,k,ncomp) ); +#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) + amrex::ignore_unused(F, coefs_y, i, j, k, ncomp); + return 0._rt; // 1D and 2D Cartesian: derivative along y is 0 +#endif + } + /** * Perform derivative along z on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -186,6 +224,26 @@ struct CartesianYeeAlgorithm { #endif } + /** + * Perform second derivative along z on a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dzz ( + T_Field const& F, + amrex::Real const * const coefs_z, int const /*n_coefs_z*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; + Real const inv_dz2 = coefs_z[0]*coefs_z[0]; +#if defined WARPX_DIM_3D + return inv_dz2*( F(i,j,k-1,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j,k+1,ncomp) ); +#elif (defined WARPX_DIM_XZ) + return inv_dz2*( F(i,j-1,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j+1,k,ncomp) ); +#elif (defined WARPX_DIM_1D_Z) + return inv_dz2*( F(i-1,j,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i+1,j,k,ncomp) ); +#endif + } + }; #endif // WARPX_FINITE_DIFFERENCE_ALGORITHM_CARTESIAN_YEE_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H index e49995557a0..436f0a83f31 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H @@ -134,6 +134,24 @@ struct CylindricalYeeAlgorithm { return inv_dr*( F(i,j,k,comp) - F(i-1,j,k,comp) ); } + /** Applies the differential operator `1/r * d(r * dF/dr)/dr`, + * where `F` is on a *cell-centered* or a nodal grid in `r` + * The input parameter `r` is given at the cell-centered position */ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dr_rDr_over_r ( + T_Field const& F, + amrex::Real const r, amrex::Real const dr, + amrex::Real const * const coefs_r, int const /*n_coefs_r*/, + int const i, int const j, int const k, int const comp ) { + + using namespace amrex; + + Real const inv_dr2 = coefs_r[0]*coefs_r[0]; + return 1._rt/r * inv_dr2*( (r+0.5_rt*dr)*(F(i+1,j,k,comp) - F(i,j,k,comp)) + - (r-0.5_rt*dr)*(F(i,j,k,comp) - F(i-1,j,k,comp)) ); + } + /** * Perform derivative along z on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -162,6 +180,21 @@ struct CylindricalYeeAlgorithm { return inv_dz*( F(i,j,k,comp) - F(i,j-1,k,comp) ); } + /** + * Perform second derivative along z on a cell-centered field `F`*/ + template< typename T_Field> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real Dzz ( + T_Field const& F, + amrex::Real const * const coefs_z, int const /*n_coefs_z*/, + int const i, int const j, int const k, int const ncomp=0 ) { + + using namespace amrex; + Real const inv_dz2 = coefs_z[0]*coefs_z[0]; + + return inv_dz2*( F(i,j-1,k,ncomp) - 2._rt*F(i,j,k,ncomp) + F(i,j+1,k,ncomp) ); + } + }; #endif // WARPX_FINITE_DIFFERENCE_ALGORITHM_CYLINDRICAL_YEE_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H index d4fdf207e52..05b1db1fe94 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H @@ -8,11 +8,9 @@ #ifndef WARPX_FIELD_ACCESSOR_FUNCTORS_H #define WARPX_FIELD_ACCESSOR_FUNCTORS_H -#include "WarpX.H" -#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" - #include - +#include +#include /** * \brief Functor that returns the division of the source m_field Array4 value diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index d045a30dd44..a7a4f10a713 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -10,6 +10,7 @@ #include "EmbeddedBoundary/WarpXFaceInfoBox_fwd.H" #include "FiniteDifferenceSolver_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "BoundaryConditions/PML_fwd.H" #include "Evolve/WarpXDtType.H" @@ -89,8 +90,8 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi); + amrex::Vector field_boundary_lo, + amrex::Vector field_boundary_hi); void ComputeDivE ( const std::array,3>& Efield, amrex::MultiFab& divE ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 23ef49b58cb..5c0c2bcc96a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -12,16 +12,16 @@ #include "HybridPICModel_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" -#include "WarpX.H" #include #include - /** * \brief This class contains the parameters needed to evaluate hybrid field * solutions (kinetic ions with fluid electrons). @@ -175,6 +175,9 @@ public: amrex::ParserExecutor<2> m_eta; bool m_resistivity_has_J_dependence = false; + /** Plasma hyper-resisitivity */ + amrex::Real m_eta_h = 0.0; + /** External current */ std::string m_Jx_ext_grid_function = "0.0"; std::string m_Jy_ext_grid_function = "0.0"; @@ -197,6 +200,12 @@ public: return current_fp_ampere[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab* + get_pointer_current_fp_external (int lev, int direction) const + { + return current_fp_external[lev][direction].get(); + } + [[nodiscard]] amrex::MultiFab* get_pointer_electron_pressure_fp (int lev) const { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8c2b5a0707a..8979036fbea 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -9,7 +9,10 @@ #include "HybridPICModel.H" +#include "FieldSolver/Fields.H" + using namespace amrex; +using namespace warpx::fields; HybridPICModel::HybridPICModel ( int nlevs_max ) { @@ -40,6 +43,8 @@ void HybridPICModel::ReadParameters () pp_hybrid.query("plasma_resistivity(rho,J)", m_eta_expression); utils::parser::queryWithParser(pp_hybrid, "n_floor", m_n_floor); + utils::parser::queryWithParser(pp_hybrid, "plasma_hyper_resistivity", m_eta_h); + // convert electron temperature from eV to J m_elec_temp *= PhysConst::q_e; @@ -95,12 +100,13 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri // the external current density multifab is made nodal to avoid needing to interpolate // to a nodal grid as has to be done for the ion and total current density multifabs + // this also allows the external current multifab to not have any ghost cells WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[x]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[x]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[y]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[y]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[z]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[z]", 0.0_rt); #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -147,15 +153,15 @@ void HybridPICModel::InitData () auto & warpx = WarpX::GetInstance(); // Get the grid staggering of the fields involved in calculating E - amrex::IntVect Jx_stag = warpx.getcurrent_fp(0,0).ixType().toIntVect(); - amrex::IntVect Jy_stag = warpx.getcurrent_fp(0,1).ixType().toIntVect(); - amrex::IntVect Jz_stag = warpx.getcurrent_fp(0,2).ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.getBfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect By_stag = warpx.getBfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.getBfield_fp(0,2).ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getEfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getEfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getEfield_fp(0,2).ixType().toIntVect(); + amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); // Check that the grid types are appropriate const bool appropriate_grids = ( @@ -222,9 +228,9 @@ void HybridPICModel::InitData () for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { #ifdef AMREX_USE_EB - auto& edge_lengths_x = warpx.getedgelengths(lev, 0); - auto& edge_lengths_y = warpx.getedgelengths(lev, 1); - auto& edge_lengths_z = warpx.getedgelengths(lev, 2); + auto& edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); + auto& edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); + auto& edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); const auto edge_lengths = std::array< std::unique_ptr, 3 >{ std::make_unique( @@ -497,7 +503,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) // charge density. if (a_dt_type == DtType::Full) { FillElectronPressureMF( - electron_pressure_fp[lev], warpx.get_pointer_rho_fp(lev) + electron_pressure_fp[lev], warpx.getFieldPointer(FieldType::rho_fp, lev) ); } else { FillElectronPressureMF( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 066e88b13c0..456c542a534 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -375,7 +375,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, - const bool include_resistivity_term ) + const bool include_resistivity_term) { // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -432,9 +432,12 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; + const auto eta_h = hybrid_model->m_eta_h; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; + const bool include_hyper_resistivity_term = (eta_h > 0.0) && include_resistivity_term; + // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations amrex::GpuArray const& Er_stag = hybrid_model->Ex_IndexType; @@ -612,6 +615,14 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val, jtot_val) * Jr(i, j, 0); } + + if (include_hyper_resistivity_term) { + // r on cell-centered point (Jr is cell-centered in r) + Real const r = rmin + (i + 0.5_rt)*dr; + + auto nabla2Jr = T_Algo::Dr_rDr_over_r(Jr, r, dr, coefs_r, n_coefs_r, i, j, 0, 0); + Er(i, j, 0) -= eta_h * nabla2Jr; + } }, // Et calculation @@ -655,16 +666,18 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val, jtot_val) * Jt(i, j, 0); } + + // Note: Hyper-resisitivity should be revisited here when modal decomposition is implemented }, // Ez calculation - [=] AMREX_GPU_DEVICE (int i, int j, int k){ + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries if (lz(i,j,0) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, 0, 0); // Interpolate current to appropriate staggering to match E field Real jtot_val = 0._rt; @@ -679,15 +692,20 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k, 0); + auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, 0, 0); // interpolate the nodal neE values to the Yee grid - auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, k, 2); + auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, 0, 2); - Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; + Ez(i, j, 0) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } + if (include_resistivity_term) { Ez(i, j, 0) += eta(rho_val, jtot_val) * Jz(i, j, 0); } + + if (include_hyper_resistivity_term) { + auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, 0, 0); + Ez(i, j, 0) -= eta_h * nabla2Jz; + } } ); @@ -726,9 +744,12 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; + const auto eta_h = hybrid_model->m_eta_h; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; + const bool include_hyper_resistivity_term = (eta_h > 0.) && include_resistivity_term; + // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations amrex::GpuArray const& Ex_stag = hybrid_model->Ex_IndexType; @@ -904,6 +925,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val, jtot_val) * Jx(i, j, k); } + + if (include_hyper_resistivity_term) { + auto nabla2Jx = T_Algo::Dxx(Jx, coefs_x, n_coefs_x, i, j, k); + Ex(i, j, k) -= eta_h * nabla2Jx; + } }, // Ey calculation @@ -943,6 +969,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val, jtot_val) * Jy(i, j, k); } + + if (include_hyper_resistivity_term) { + auto nabla2Jy = T_Algo::Dyy(Jy, coefs_y, n_coefs_y, i, j, k); + Ey(i, j, k) -= eta_h * nabla2Jy; + } }, // Ez calculation @@ -976,6 +1007,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Add resistivity only if E field value is used to update B if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } + + if (include_hyper_resistivity_term) { + auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, k); + Ez(i, j, k) -= eta_h * nabla2Jz; + } } ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 205b009b2eb..129b2e2f782 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -13,8 +13,11 @@ #include "Utils/WarpXConst.H" #include +#include +#include #include #include +#include #include #include #include @@ -33,8 +36,26 @@ public: MacroscopicProperties (); // constructor /** Read user-defined macroscopic properties. Called in constructor. */ void ReadParameters (); - /** Initialize multifabs storing macroscopic multifabs */ - void InitData (); + + /** + * \brief Initialize multifabs storing macroscopic multifabs + * + * @param[in] ba the box array associated to the multifabs E and B + * @param[in] dmap the distribution mapping + * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B + * @param[in] geom the geometry + * @param[in] Ex_stag staggering of the Ex field + * @param[in] Ey_stag staggering of the Ey field + * @param[in] Ez_stag staggering of the Ez field + */ + void InitData ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dmap, + const amrex::IntVect& ng_EB_alloc, + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag); /** return MultiFab, sigma (conductivity) of the medium. */ amrex::MultiFab& getsigma_mf () {return (*m_sigma_mf);} diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 965db68a558..1467c7a8c0c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -1,19 +1,16 @@ #include "MacroscopicProperties.H" +#include "FieldSolver/Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" -#include "WarpX.H" #include #include -#include #include -#include #include #include #include -#include #include #include #include @@ -26,6 +23,7 @@ #include using namespace amrex; +using namespace warpx::fields; MacroscopicProperties::MacroscopicProperties () { @@ -121,16 +119,17 @@ MacroscopicProperties::ReadParameters () } void -MacroscopicProperties::InitData () +MacroscopicProperties::InitData ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dmap, + const amrex::IntVect& ng_EB_alloc, + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag) { amrex::Print() << Utils::TextMsg::Info("we are in init data of macro"); - auto & warpx = WarpX::GetInstance(); - // Get BoxArray and DistributionMap of warpx instance. - const int lev = 0; - const amrex::BoxArray ba = warpx.boxArray(lev); - const amrex::DistributionMapping dmap = warpx.DistributionMap(lev); - const amrex::IntVect ng_EB_alloc = warpx.getngEB(); // Define material property multifabs using ba and dmap from WarpX instance // sigma is cell-centered MultiFab m_sigma_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); @@ -146,7 +145,7 @@ MacroscopicProperties::InitData () } else if (m_sigma_s == "parse_sigma_function") { InitializeMacroMultiFabUsingParser(m_sigma_mf.get(), m_sigma_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } // Initialize epsilon if (m_epsilon_s == "constant") { @@ -156,7 +155,7 @@ MacroscopicProperties::InitData () } else if (m_epsilon_s == "parse_epsilon_function") { InitializeMacroMultiFabUsingParser(m_eps_mf.get(), m_epsilon_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } // In the Maxwell solver, `epsilon` is used in the denominator. @@ -173,16 +172,14 @@ MacroscopicProperties::InitData () } else if (m_mu_s == "parse_mu_function") { InitializeMacroMultiFabUsingParser(m_mu_mf.get(), m_mu_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } - amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); - amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); - amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getEfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getEfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getEfield_fp(0,2).ixType().toIntVect(); + const amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); + const amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); + const amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { sigma_IndexType[idim] = sigma_stag[idim]; diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H index f84a8f963fc..a8bbc954e29 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef MAGNETOSTATICSOLVER_H_ -#define MAGNETOSTATICSOLVER_H_ +#ifndef WARPX_MAGNETOSTATICSOLVER_H_ +#define WARPX_MAGNETOSTATICSOLVER_H_ #include #include @@ -54,4 +54,4 @@ namespace MagnetostaticSolver { }; } // namespace MagnetostaticSolver -#endif //MAGNETOSTATICSOLVER_H_ +#endif //WARPX_MAGNETOSTATICSOLVER_H_ diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 192017656ce..26ac1ac96c8 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -318,9 +318,9 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs dirichlet_flag[adim][idim*2] = false; } else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary conditions have to be either periodic, PEC, or Neumann " - "when using the magnetostatic solver" + WARPX_ABORT_WITH_MESSAGE( + "Field boundary conditions have to be either periodic, PEC or neumann " + "when using the magnetostatic solver, but they are " + GetFieldBCTypeString(WarpX::field_boundary_lo[idim]) ); } @@ -338,9 +338,9 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs dirichlet_flag[adim][idim*2+1] = false; } else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary conditions have to be either periodic, PEC, or Neumann " - "when using the magnetostatic solver" + WARPX_ABORT_WITH_MESSAGE( + "Field boundary conditions have to be either periodic, PEC or neumann " + "when using the magnetostatic solver, but they are " + GetFieldBCTypeString(WarpX::field_boundary_lo[idim]) ); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 53b8e67412d..0d29cdca74a 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -20,6 +20,7 @@ # include "FieldSolver/SpectralSolver/SpectralSolver.H" # endif #endif +#include "Python/callbacks.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -798,6 +799,9 @@ WarpX::EvolveB (amrex::Real a_dt, DtType a_dt_type) for (int lev = 0; lev <= finest_level; ++lev) { EvolveB(lev, a_dt, a_dt_type); } + + // Allow execution of Python callback after B-field push + ExecutePythonCallback("afterBpush"); } void @@ -848,6 +852,9 @@ WarpX::EvolveE (amrex::Real a_dt) { EvolveE(lev, a_dt); } + + // Allow execution of Python callback after E-field push + ExecutePythonCallback("afterEpush"); } void diff --git a/Source/FieldSolver/WarpXPushFieldsEM_K.H b/Source/FieldSolver/WarpXPushFieldsEM_K.H index d6e7e353a54..4ff7b32b517 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM_K.H +++ b/Source/FieldSolver/WarpXPushFieldsEM_K.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef WarpXPushFieldsEM_K_h -#define WarpXPushFieldsEM_K_h +#ifndef WARPX_PushFieldsEM_K_h +#define WARPX_PushFieldsEM_K_h #include "Utils/WarpXConst.H" @@ -117,4 +117,4 @@ void damp_field_in_guards( } } -#endif +#endif //WARPX_PushFieldsEM_K_h diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index ae10ae1e19a..4dbe10c4e5a 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -154,8 +154,7 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, rho_fp, m_edge_lengths, - false + Efield_fp, current_fp_temp, Bfield_fp, rho_fp, m_edge_lengths, false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); diff --git a/Source/FieldSolver/WarpX_QED_K.H b/Source/FieldSolver/WarpX_QED_K.H index 6c86bba67b6..eccb8d6e72d 100644 --- a/Source/FieldSolver/WarpX_QED_K.H +++ b/Source/FieldSolver/WarpX_QED_K.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef WarpX_QED_K_h -#define WarpX_QED_K_h +#ifndef WARPX_QED_K_h +#define WARPX_QED_K_h #include "Utils/WarpXConst.H" @@ -359,4 +359,4 @@ const amrex::Real dyi = 1._rt/dy; } -#endif +#endif //WARPX_QED_K_h diff --git a/Source/Fluids/MultiFluidContainer_fwd.H b/Source/Fluids/MultiFluidContainer_fwd.H index a1a55c3b450..01edf7f728c 100644 --- a/Source/Fluids/MultiFluidContainer_fwd.H +++ b/Source/Fluids/MultiFluidContainer_fwd.H @@ -6,6 +6,7 @@ */ #ifndef WARPX_MultiFluidContainer_fwd_H_ +#define WARPX_MultiFluidContainer_fwd_H_ class MultiFluidContainer; diff --git a/Source/Fluids/WarpXFluidContainer_fwd.H b/Source/Fluids/WarpXFluidContainer_fwd.H index 77a6bc0c991..86fbff98fd0 100644 --- a/Source/Fluids/WarpXFluidContainer_fwd.H +++ b/Source/Fluids/WarpXFluidContainer_fwd.H @@ -6,6 +6,7 @@ */ #ifndef WARPX_WarpXFluidContainer_fwd_H_ +#define WARPX_WarpXFluidContainer_fwd_H_ class WarpXFluidContainer; diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H index 7b64339a042..b32e361bff4 100644 --- a/Source/Initialization/ExternalField.H +++ b/Source/Initialization/ExternalField.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef EXTERNAL_FIELD_H_ -#define EXTERNAL_FIELD_H_ +#ifndef WARPX_EXTERNAL_FIELD_H_ +#define WARPX_EXTERNAL_FIELD_H_ #include "ExternalField_fwd.H" @@ -66,4 +66,4 @@ struct ExternalFieldParams std::string external_fields_path; }; -#endif //EXTERNAL_FIELD_H_ +#endif //WARPX_EXTERNAL_FIELD_H_ diff --git a/Source/Initialization/ExternalField_fwd.H b/Source/Initialization/ExternalField_fwd.H index f7a8547ecce..a2fd6471222 100644 --- a/Source/Initialization/ExternalField_fwd.H +++ b/Source/Initialization/ExternalField_fwd.H @@ -5,9 +5,9 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef EXTERNAL_FIELD_FWD_H_ -#define EXTERNAL_FIELD_FWD_H_ +#ifndef WARPX_EXTERNAL_FIELD_FWD_H_ +#define WARPX_EXTERNAL_FIELD_FWD_H_ struct ExternalFieldParams; -#endif //EXTERNAL_FIELD_FWD_H_ +#endif //WARPX_EXTERNAL_FIELD_FWD_H_ diff --git a/Source/Initialization/GetTemperature.H b/Source/Initialization/GetTemperature.H index 4437928697c..20760863fae 100644 --- a/Source/Initialization/GetTemperature.H +++ b/Source/Initialization/GetTemperature.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef GET_TEMPERATURE_H_ -#define GET_TEMPERATURE_H_ +#ifndef WARPX_GET_TEMPERATURE_H_ +#define WARPX_GET_TEMPERATURE_H_ #include "TemperatureProperties.H" @@ -69,4 +69,4 @@ struct GetTemperature } } }; -#endif +#endif //WARPX_GET_TEMPERATURE_H_ diff --git a/Source/Initialization/GetVelocity.H b/Source/Initialization/GetVelocity.H index 3123708bf06..c8535b8af8e 100644 --- a/Source/Initialization/GetVelocity.H +++ b/Source/Initialization/GetVelocity.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef GET_VELOCITY_H_ -#define GET_VELOCITY_H_ +#ifndef WARPX_GET_VELOCITY_H_ +#define WARPX_GET_VELOCITY_H_ #include "VelocityProperties.H" @@ -86,4 +86,4 @@ struct GetVelocity return m_dir; } }; -#endif +#endif //WARPX_GET_VELOCITY_H_ diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H index 3848a42f4ee..465627f874a 100644 --- a/Source/Initialization/InjectorDensity.H +++ b/Source/Initialization/InjectorDensity.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_DENSITY_H_ -#define INJECTOR_DENSITY_H_ +#ifndef WARPX_INJECTOR_DENSITY_H_ +#define WARPX_INJECTOR_DENSITY_H_ #include "Utils/WarpXConst.H" @@ -218,4 +218,4 @@ struct InjectorDensityDeleter { } }; -#endif +#endif //WARPX_INJECTOR_DENSITY_H_ diff --git a/Source/Initialization/InjectorFlux.H b/Source/Initialization/InjectorFlux.H index e889c8b8966..6d608822c31 100644 --- a/Source/Initialization/InjectorFlux.H +++ b/Source/Initialization/InjectorFlux.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_FLUX_H_ -#define INJECTOR_FLUX_H_ +#ifndef WARPX_INJECTOR_FLUX_H_ +#define WARPX_INJECTOR_FLUX_H_ #include "Utils/WarpXConst.H" @@ -149,4 +149,4 @@ struct InjectorFluxDeleter { } }; -#endif +#endif //WARPX_INJECTOR_FLUX_H_ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index f7ee6cff138..52fedcd779f 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -5,13 +5,14 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_MOMENTUM_H_ -#define INJECTOR_MOMENTUM_H_ +#ifndef WARPX_INJECTOR_MOMENTUM_H_ +#define WARPX_INJECTOR_MOMENTUM_H_ #include "GetTemperature.H" #include "GetVelocity.H" #include "TemperatureProperties.H" #include "VelocityProperties.H" +#include "SampleGaussianFluxDistribution.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -90,75 +91,6 @@ private: amrex::Real m_ux_th, m_uy_th, m_uz_th; }; -namespace { - /** Return u sampled according to the probability distribution: - * p(u) \propto u \exp(-(u-u_m)^2/2u_th^2) - * - * @param u_m Central momentum - * @param u_th Momentum spread - * @param engine Object used to generate random numbers - */ - [[nodiscard]] - AMREX_FORCE_INLINE - AMREX_GPU_HOST_DEVICE - amrex::Real - generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { - - using namespace amrex::literals; - - // Momentum to be returned at the end of this function - amrex::Real u = 0._rt; - - const amrex::Real abs_u_m = std::abs(u_m); - - if (u_th == 0._rt) { - u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below - } else if (abs_u_m < 0.6*u_th) { - // Mean velocity magnitude is less than thermal velocity - // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation - // and then use the rejection method to correct it - // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) - // Note that this is the method that is used in the common case u_m=0 - const amrex::Real umsign = std::copysign(1._rt, u_m); - const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); - const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation - bool reject = true; - while (reject) { - // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), - // using the method of the inverse cumulative function - amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 - u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); - // Rejection method - xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } - } - } else { - // Mean velocity magnitude is greater than thermal velocity - // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation - // and then use the rejection method to correct it - // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note - // that this number is always between 0 and 1 ) - // Note that in the common case `u_m = 0`, this rejection method - // is not used, and the above rejection method is used instead. - bool reject = true; - const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; - const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation - while (reject) { - // Approximate distribution: normal distribution, where we only retain positive u - u = -1._rt; - while (u < 0) { - u = amrex::RandomNormal(approx_u_m, u_th, engine); - } - // Rejection method - const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } - } - } - - return u; - } -} - // struct whose getMomentum returns momentum for 1 particle, from random // gaussian flux distribution in the specified direction. // Along the normal axis, the distribution is v*Gaussian, @@ -839,4 +771,4 @@ struct InjectorMomentumDeleter { } }; -#endif +#endif //WARPX_INJECTOR_MOMENTUM_H_ diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 7168ac93ebb..1b3be9e28a7 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_POSITION_H_ -#define INJECTOR_POSITION_H_ +#ifndef WARPX_INJECTOR_POSITION_H_ +#define WARPX_INJECTOR_POSITION_H_ #include "InjectorPosition_fwd.H" @@ -255,4 +255,4 @@ private: amrex::Real zmin, zmax; }; -#endif +#endif //WARPX_INJECTOR_POSITION_H_ diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 2651aad455f..b9fe2323290 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PLASMA_INJECTOR_H_ -#define PLASMA_INJECTOR_H_ +#ifndef WARPX_PLASMA_INJECTOR_H_ +#define WARPX_PLASMA_INJECTOR_H_ #include "InjectorDensity.H" #include "InjectorFlux.H" @@ -202,4 +202,4 @@ protected: void parseFlux (amrex::ParmParse const& pp_species); }; -#endif +#endif //WARPX_PLASMA_INJECTOR_H_ diff --git a/Source/Initialization/SampleGaussianFluxDistribution.H b/Source/Initialization/SampleGaussianFluxDistribution.H new file mode 100644 index 00000000000..65cad141e8f --- /dev/null +++ b/Source/Initialization/SampleGaussianFluxDistribution.H @@ -0,0 +1,81 @@ +/* Copyright 2024 Remi Lehe, Revathi Jambunathan + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#define WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H + +#include + +namespace { + /** This function returns u sampled according to the probability distribution: + * p(u) \propto u \exp(-(u-u_m)^2/2u_th^2) + * + * @param u_m Central momentum + * @param u_th Momentum spread + * @param engine Object used to generate random numbers + */ + [[nodiscard]] + AMREX_FORCE_INLINE + AMREX_GPU_HOST_DEVICE + amrex::Real + generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { + + using namespace amrex::literals; + + // Momentum to be returned at the end of this function + amrex::Real u = 0._rt; + + const amrex::Real abs_u_m = std::abs(u_m); + + if (u_th == 0._rt) { + u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below + } else if (abs_u_m < 0.6*u_th) { + // Mean velocity magnitude is less than thermal velocity + // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation + // and then use the rejection method to correct it + // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) + // Note that this is the method that is used in the common case u_m=0 + const amrex::Real umsign = std::copysign(1._rt, u_m); + const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); + const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation + bool reject = true; + while (reject) { + // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), + // using the method of the inverse cumulative function + amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 + u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); + // Rejection method + xrand = amrex::Random(engine); + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } + } + } else { + // Mean velocity magnitude is greater than thermal velocity + // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation + // and then use the rejection method to correct it + // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note + // that this number is always between 0 and 1 ) + // Note that in the common case `u_m = 0`, this rejection method + // is not used, and the above rejection method is used instead. + bool reject = true; + const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; + const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation + while (reject) { + // Approximate distribution: normal distribution, where we only retain positive u + u = -1._rt; + while (u < 0) { + u = amrex::RandomNormal(approx_u_m, u_th, engine); + } + // Rejection method + const amrex::Real xrand = amrex::Random(engine); + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } + } + } + return u; + } +} + +#endif //WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H diff --git a/Source/Initialization/TemperatureProperties.H b/Source/Initialization/TemperatureProperties.H index 4d30b67c851..78a967a32fa 100644 --- a/Source/Initialization/TemperatureProperties.H +++ b/Source/Initialization/TemperatureProperties.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TEMPERATURE_PROPERTIES_H_ -#define TEMPERATURE_PROPERTIES_H_ +#ifndef WARPX_TEMPERATURE_PROPERTIES_H_ +#define WARPX_TEMPERATURE_PROPERTIES_H_ #include #include @@ -43,4 +43,4 @@ struct TemperatureProperties std::unique_ptr m_ptr_temperature_parser; }; -#endif +#endif //WARPX_TEMPERATURE_PROPERTIES_H_ diff --git a/Source/Initialization/VelocityProperties.H b/Source/Initialization/VelocityProperties.H index 453d97725e2..87730acbc3a 100644 --- a/Source/Initialization/VelocityProperties.H +++ b/Source/Initialization/VelocityProperties.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef VELOCITY_PROPERTIES_H_ -#define VELOCITY_PROPERTIES_H_ +#ifndef WARPX_VELOCITY_PROPERTIES_H_ +#define WARPX_VELOCITY_PROPERTIES_H_ #include #include @@ -50,4 +50,4 @@ struct VelocityProperties std::unique_ptr m_ptr_velocity_parser; }; -#endif +#endif //WARPX_VELOCITY_PROPERTIES_H_ diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 613250b5c2f..ab65fa2687c 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -33,4 +33,4 @@ namespace warpx::initialization ); } -#endif +#endif //WARPX_AMREX_INIT_H_ diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index d1dd0bbe90b..4036a49235c 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include namespace { /** Overwrite defaults in AMReX Inputs diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 4963c66077b..6b0cba707fe 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -217,11 +217,20 @@ WarpX::PrintMainPICparameters () if ( (em_solver_medium == MediumForEM::Macroscopic) && (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::LaxWendroff)){ amrex::Print() << " | - Lax-Wendroff algorithm\n"; - } + } else if ((em_solver_medium == MediumForEM::Macroscopic) && (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::BackwardEuler)){ amrex::Print() << " | - Backward Euler algorithm\n"; - } + } + if(electrostatic_solver_id != ElectrostaticSolverAlgo::None){ + if(poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction){ + amrex::Print() << "Poisson solver: | FFT-based" << "\n"; + } + else if(poisson_solver_id == PoissonSolverAlgo::Multigrid){ + amrex::Print() << "Poisson solver: | multigrid" << "\n"; + } + } + amrex::Print() << "-------------------------------------------------------------------------------\n"; // Print type of current deposition if (current_deposition_algo == CurrentDepositionAlgo::Direct){ @@ -304,6 +313,18 @@ WarpX::PrintMainPICparameters () amrex::Print() << " | - multi-J deposition is ON \n"; amrex::Print() << " | - do_multi_J_n_depositions = " << WarpX::do_multi_J_n_depositions << "\n"; + if (J_in_time == JInTime::Linear){ + amrex::Print() << " | - J_in_time = linear \n"; + } + if (J_in_time == JInTime::Constant){ + amrex::Print() << " | - J_in_time = constant \n"; + } + if (rho_in_time == RhoInTime::Linear){ + amrex::Print() << " | - rho_in_time = linear \n"; + } + if (rho_in_time == RhoInTime::Constant){ + amrex::Print() << " | - rho_in_time = constant \n"; + } } if (fft_do_time_averaging == 1){ amrex::Print()<<" | - time-averaged is ON \n"; @@ -457,7 +478,16 @@ WarpX::InitData () BuildBufferMasks(); if (WarpX::em_solver_medium==1) { - m_macroscopic_properties->InitData(); + const int lev_zero = 0; + m_macroscopic_properties->InitData( + boxArray(lev_zero), + DistributionMap(lev_zero), + getngEB(), + Geom(lev_zero), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + ); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { @@ -570,16 +600,17 @@ WarpX::InitPML () // Note: fill_guards_fields and fill_guards_current are both set to // zero (amrex::IntVect(0)) (what we do with damping BCs does not apply // to the PML, for example in the presence of mesh refinement patches) - pml[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), nullptr, - pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), - dt[0], nox_fft, noy_fft, noz_fft, grid_type, - do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, - do_pml_dive_cleaning, do_pml_divb_cleaning, - amrex::IntVect(0), amrex::IntVect(0), - guard_cells.ng_FieldSolver.max(), - v_particle_pml, - do_pml_Lo[0], do_pml_Hi[0]); + pml[0] = std::make_unique( + 0, boxArray(0), DistributionMap(0), do_similar_dm_pml, &Geom(0), nullptr, + pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), + dt[0], nox_fft, noy_fft, noz_fft, grid_type, + do_moving_window, pml_has_particles, do_pml_in_domain, + psatd_solution_type, J_in_time, rho_in_time, + do_pml_dive_cleaning, do_pml_divb_cleaning, + amrex::IntVect(0), amrex::IntVect(0), + guard_cells.ng_FieldSolver.max(), + v_particle_pml, + do_pml_Lo[0], do_pml_Hi[0]); #endif for (int lev = 1; lev <= finest_level; ++lev) @@ -608,16 +639,17 @@ WarpX::InitPML () // Note: fill_guards_fields and fill_guards_current are both set to // zero (amrex::IntVect(0)) (what we do with damping BCs does not apply // to the PML, for example in the presence of mesh refinement patches) - pml[lev] = std::make_unique(lev, boxArray(lev), DistributionMap(lev), - &Geom(lev), &Geom(lev-1), - pml_ncell, pml_delta, refRatio(lev-1), - dt[lev], nox_fft, noy_fft, noz_fft, grid_type, - do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, - amrex::IntVect(0), amrex::IntVect(0), - guard_cells.ng_FieldSolver.max(), - v_particle_pml, - do_pml_Lo[lev], do_pml_Hi[lev]); + pml[lev] = std::make_unique( + lev, boxArray(lev), DistributionMap(lev), do_similar_dm_pml, + &Geom(lev), &Geom(lev-1), + pml_ncell, pml_delta, refRatio(lev-1), + dt[lev], nox_fft, noy_fft, noz_fft, grid_type, + do_moving_window, pml_has_particles, do_pml_in_domain, + psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, + amrex::IntVect(0), amrex::IntVect(0), + guard_cells.ng_FieldSolver.max(), + v_particle_pml, + do_pml_Lo[lev], do_pml_Hi[lev]); } } } @@ -1034,7 +1066,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1069,7 +1101,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1100,7 +1132,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1318,12 +1350,6 @@ void WarpX::CheckKnownIssues() ablastr::warn_manager::WarnPriority::low); } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !load_balance_intervals.isActivated(), - "The hybrid-PIC algorithm involves multifabs that are not yet " - "properly redistributed during load balancing events." - ); - const bool external_particle_field_used = ( mypc->m_B_ext_particle_s != "none" || mypc->m_E_ext_particle_s != "none" ); @@ -1379,8 +1405,8 @@ WarpX::LoadExternalFieldsFromFile (int const lev) #if defined(WARPX_USE_OPENPMD) && !defined(WARPX_DIM_1D_Z) && !defined(WARPX_DIM_XZ) void WarpX::ReadExternalFieldFromFile ( - std::string read_fields_from_path, amrex::MultiFab* mf, - std::string F_name, std::string F_component) + const std::string& read_fields_from_path, amrex::MultiFab* mf, + const std::string& F_name, const std::string& F_component) { // Get WarpX domain info auto& warpx = WarpX::GetInstance(); @@ -1556,7 +1582,7 @@ WarpX::ReadExternalFieldFromFile ( } // End function WarpX::ReadExternalFieldFromFile #else // WARPX_USE_OPENPMD && !WARPX_DIM_1D_Z && !defined(WARPX_DIM_XZ) void -WarpX::ReadExternalFieldFromFile (std::string , amrex::MultiFab* ,std::string, std::string) +WarpX::ReadExternalFieldFromFile (const std::string& , amrex::MultiFab* , const std::string& , const std::string& ) { #if defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE("Reading fields from openPMD files is not supported in 1D"); diff --git a/Source/Laser/LaserProfiles.H b/Source/Laser/LaserProfiles.H index 3aea88ac956..c2a3f337fbf 100644 --- a/Source/Laser/LaserProfiles.H +++ b/Source/Laser/LaserProfiles.H @@ -286,7 +286,7 @@ private: /** \brief parse a field file in the HDF5 'lasy' format * \param lasy_file_name: name of the lasy file to parse */ - void parse_lasy_file(std::string lasy_file_name); + void parse_lasy_file(const std::string& lasy_file_name); /** \brief parse a field file in the binary 'binary' format (whose details are given below). * @@ -301,7 +301,7 @@ private: * The spatiotemporal grid must be rectangular and uniform. * \param binary_file_name: name of the binary file to parse */ - void parse_binary_file(std::string binary_file_name); + void parse_binary_file(const std::string& binary_file_name); /** \brief Finds left and right time indices corresponding to time t * diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index 0fdb45c64f8..934a537be5a 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -162,7 +162,7 @@ WarpXLaserProfiles::FromFileLaserProfile::fill_amplitude ( } void -WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_name) +WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(const std::string& lasy_file_name) { #ifdef WARPX_USE_OPENPMD if(ParallelDescriptor::IOProcessor()){ @@ -237,7 +237,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ } void -WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_file_name) +WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (const std::string& binary_file_name) { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 1a77a68fa58..9051c3598e6 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -216,11 +216,6 @@ ifeq ($(USE_HDF5),TRUE) DEFINES += -DWARPX_USE_HDF5 endif -ifeq ($(USE_GPUCLOCK),TRUE) - USERSuffix := $(USERSuffix).GPUCLOCK - DEFINES += -DWARPX_USE_GPUCLOCK -endif - # job_info support CEXE_sources += AMReX_buildInfo.cpp INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 38cef54921b..341db01bef6 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef GUARDCELLMANAGER_H_ -#define GUARDCELLMANAGER_H_ +#ifndef WARPX_GUARDCELLMANAGER_H_ +#define WARPX_GUARDCELLMANAGER_H_ #include #include @@ -63,8 +63,8 @@ public: int nci_corr_stencil, int electromagnetic_solver_id, int max_level, - amrex::Vector v_galilean, - amrex::Vector v_comoving, + const amrex::Vector& v_galilean, + const amrex::Vector& v_comoving, bool safe_guard_cells, int do_multi_J, bool fft_do_time_averaging, @@ -108,4 +108,4 @@ public: amrex::IntVect ng_depos_rho = amrex::IntVect::TheZeroVector(); }; -#endif // GUARDCELLMANAGER_H_ +#endif // WARPX_GUARDCELLMANAGER_H_ diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index ae5ab839d10..28157e09d8c 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -44,8 +44,8 @@ guardCellManager::Init ( const int nci_corr_stencil, const int electromagnetic_solver_id, const int max_level, - const amrex::Vector v_galilean, - const amrex::Vector v_comoving, + const amrex::Vector& v_galilean, + const amrex::Vector& v_comoving, const bool safe_guard_cells, const int do_multi_J, const bool fft_do_time_averaging, @@ -120,9 +120,11 @@ guardCellManager::Init ( #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) ng_alloc_EB = IntVect(ngx,ngz); ng_alloc_J = IntVect(ngJx,ngJz); + amrex::ignore_unused(ngy, ngJy); #elif defined(WARPX_DIM_1D_Z) ng_alloc_EB = IntVect(ngz); ng_alloc_J = IntVect(ngJz); + amrex::ignore_unused(ngx, ngJx, ngy, ngJy); #endif // TODO Adding one cell for rho should not be necessary, given that the number of guard cells diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 7d5b25b1560..f3bff69546b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -596,7 +596,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In for (int i = 0; i < 3; ++i) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= mf[i]->nGrowVect(), + ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; @@ -653,7 +653,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In for (int i = 0; i < 3; ++i) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= mf[i]->nGrowVect(), + ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; @@ -684,7 +684,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Efield_avg_fp[lev][0]->nGrowVect(), + ng.allLE(Efield_avg_fp[lev][0]->nGrowVect()), "Error: in FillBoundaryE_avg, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Efield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Efield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -705,7 +705,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Efield_avg_cp[lev][0]->nGrowVect(), + ng.allLE(Efield_avg_cp[lev][0]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Efield_avg_cp[lev][0], ng, WarpX::do_single_precision_comms, cperiod); ablastr::utils::communication::FillBoundary(*Efield_avg_cp[lev][1], ng, WarpX::do_single_precision_comms, cperiod); @@ -737,7 +737,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Bfield_fp[lev][0]->nGrowVect(), + ng.allLE(Bfield_fp[lev][0]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -757,7 +757,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Bfield_avg_cp[lev][0]->nGrowVect(), + ng.allLE(Bfield_avg_cp[lev][0]->nGrowVect()), "Error: in FillBoundaryB_avg, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_cp[lev][0], ng, WarpX::do_single_precision_comms, cperiod); ablastr::utils::communication::FillBoundary(*Bfield_avg_cp[lev][1], ng, WarpX::do_single_precision_comms, cperiod); @@ -882,7 +882,7 @@ WarpX::SyncCurrent ( WARPX_PROFILE("WarpX::SyncCurrent()"); // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid - if (WarpX::do_current_centering) + if (do_current_centering) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); @@ -1178,7 +1178,7 @@ void WarpX::SumBoundaryJ ( const amrex::IntVect ng = J.nGrowVect(); amrex::IntVect ng_depos_J = get_ng_depos_J(); - if (WarpX::do_current_centering) + if (do_current_centering) { #if defined(WARPX_DIM_1D_Z) ng_depos_J[0] += WarpX::current_centering_noz / 2; diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index 4cec33bc84f..a2b8fe38ed4 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -179,10 +179,6 @@ void warpx_interp (int j, int k, int l, amrex::Real fine = 0.0_rt; amrex::Real coarse = 0.0_rt; - amrex::Real wj; - amrex::Real wk; - amrex::Real wl; - // 1) Interpolation from coarse nodal to fine nodal nj = 2; @@ -197,20 +193,18 @@ void warpx_interp (int j, int k, int l, nl = 2; #endif - wj = 1.0_rt; - wk = 1.0_rt; - wl = 1.0_rt; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); + auto c = arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + c *= (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); #if (AMREX_SPACEDIM >= 2) - wk = (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); + c *= (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); #endif #if (AMREX_SPACEDIM == 3) - wl = (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); + c *= (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); #endif - tmp += wj * wk * wl * arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + tmp += c; } } } @@ -237,20 +231,18 @@ void warpx_interp (int j, int k, int l, kc = amrex::coarsen(kn, rk); lc = amrex::coarsen(ln, rl); - wj = 1.0_rt; - wk = 1.0_rt; - wl = 1.0_rt; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); + auto c = arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + c *= (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); #if (AMREX_SPACEDIM >= 2) - wk = (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); + c *= (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); #endif #if (AMREX_SPACEDIM == 3) - wl = (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); + c *= (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); #endif - coarse += wj * wk * wl * arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + coarse += c; } } } @@ -284,13 +276,11 @@ void warpx_interp (int j, int k, int l, for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = 1.0_rt / static_cast(nj); - wk = 1.0_rt / static_cast(nk); - wl = 1.0_rt / static_cast(nl); - fine += wj * wk * wl * arr_fine_zeropad(jm+jj,km+kk,lm+ll); + fine += arr_fine_zeropad(jm+jj,km+kk,lm+ll); } } } + fine = fine/static_cast(nj*nk*nl); // Final result arr_aux(j,k,l) = tmp + (fine - coarse); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index cfd8bf22c2b..b2a7004a1af 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -11,6 +11,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" @@ -49,12 +50,29 @@ using namespace amrex; +void +WarpX::CheckLoadBalance (int step) +{ + if (step > 0 && load_balance_intervals.contains(step+1)) + { + LoadBalance(); + + // Reset the costs to 0 + ResetCosts(); + } + if (!costs.empty()) + { + RescaleCosts(step); + } +} + void WarpX::LoadBalance () { WARPX_PROFILE_REGION("LoadBalance"); WARPX_PROFILE("WarpX::LoadBalance()"); + AMREX_ALWAYS_ASSERT(!costs.empty()); AMREX_ALWAYS_ASSERT(costs[0] != nullptr); #ifdef AMREX_USE_MPI @@ -191,6 +209,11 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(Efield_avg_fp[lev][idim], dm, true ,lev); RemakeMultiFab(Bfield_avg_fp[lev][idim], dm, true ,lev); } + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], dm, false, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim], dm, true, lev); + } #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { RemakeMultiFab(m_edge_lengths[lev][idim], dm, false ,lev); @@ -213,6 +236,11 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // the last step as the initial guess for the next solve RemakeMultiFab(phi_fp[lev], dm, true ,lev); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], dm, false, lev); + } + #ifdef AMREX_USE_EB RemakeMultiFab(m_distance_to_eb[lev], dm, false ,lev); @@ -403,6 +431,9 @@ WarpX::ComputeCostsHeuristic (amrex::VectorIndexArray(); @@ -413,3 +444,30 @@ WarpX::ResetCosts () } } } + +void +WarpX::RescaleCosts (int step) +{ + // rescale is only used for timers + if (WarpX::load_balance_costs_update_algo != LoadBalanceCostsUpdateAlgo::Timers) + { + return; + } + + AMREX_ALWAYS_ASSERT(costs.size() == finest_level + 1); + + for (int lev = 0; lev <= finest_level; ++lev) + { + if (costs[lev]) + { + // Perform running average of the costs + // (Giving more importance to most recent costs; only needed + // for timers update, heuristic load balance considers the + // instantaneous costs) + for (const auto& i : costs[lev]->IndexArray()) + { + (*costs[lev])[i] *= (1._rt - 2._rt/load_balance_intervals.localPeriod(step+1)); + } + } + } +} diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 260ebb3871a..9c7c2e60160 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -16,11 +16,7 @@ * This is typically called for the sources of the Maxwell equations (J/rho) * after deposition from the macroparticles. * - * - When WarpX is used with a finite-difference scheme: this only - * updates the *valid* cells of `mf` - * - When WarpX is used with a spectral scheme (PSATD): this - * updates both the *valid* cells and *guard* cells. (This is because a - * spectral solver requires the value of the sources over a large stencil.) + * This updates both the *valid* cells and *guard* cells. */ void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, @@ -33,11 +29,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * This is typically called for the sources of the Maxwell equations (J/rho) * after deposition from the macroparticles + filtering. * - * - When WarpX is used with a finite-difference scheme: this only - * updates the *valid* cells of `dst` - * - When WarpX is used with a spectral scheme (PSATD): this - * updates both the *valid* cells and *guard* cells. (This is because a - * spectral solver requires the value of the sources over a large stencil.) + * This updates both the *valid* cells and *guard* cells. * * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp index 20ac73cab50..e752d49eedf 100644 --- a/Source/Parallelization/WarpXSumGuardCells.cpp +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -19,14 +19,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, const int icomp, const int ncomp) { - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = mf.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } + amrex::IntVect const n_updated_guards = mf.nGrowVect(); ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); } @@ -37,14 +30,7 @@ WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::IntVect& src_ngrow, const int icomp, const int ncomp) { - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = dst.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } + amrex::IntVect const n_updated_guards = dst.nGrowVect(); dst.setVal(0., icomp, ncomp, n_updated_guards); dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); diff --git a/Source/Particles/Algorithms/KineticEnergy.H b/Source/Particles/Algorithms/KineticEnergy.H index 87da83f9d8d..f097b6436e1 100644 --- a/Source/Particles/Algorithms/KineticEnergy.H +++ b/Source/Particles/Algorithms/KineticEnergy.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLES_KINETIC_ENERGY_H_ -#define PARTICLES_KINETIC_ENERGY_H_ +#ifndef WARPX_PARTICLES_KINETIC_ENERGY_H_ +#define WARPX_PARTICLES_KINETIC_ENERGY_H_ #include "Utils/WarpXConst.H" @@ -68,4 +68,4 @@ namespace Algorithms{ } -#endif // PARTICLES_ALGORITHMS_H_ +#endif // WARPX_PARTICLES_KINETIC_ENERGY_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index 1de6b999e0b..3fd8b4d6fc2 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -23,7 +23,7 @@ class BackgroundMCCCollision final : public CollisionBase { public: - BackgroundMCCCollision (std::string collision_name); + BackgroundMCCCollision (std::string const& collision_name); ~BackgroundMCCCollision () override = default; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 4cb16f6fa50..1eb89ff4c0f 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -21,7 +21,7 @@ #include -BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name) +BackgroundMCCCollision::BackgroundMCCCollision (std::string const& collision_name) : CollisionBase(collision_name) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_species_names.size() == 1, diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H index 55fa4b9e1e3..8ba9b6f1b31 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H @@ -24,7 +24,7 @@ class BackgroundStopping final : public CollisionBase { public: - BackgroundStopping (std::string collision_name); + BackgroundStopping (const std::string& collision_name); ~BackgroundStopping () override = default; diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index 8122d7225b4..517b1138295 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -16,7 +16,7 @@ #include -BackgroundStopping::BackgroundStopping (std::string const collision_name) +BackgroundStopping::BackgroundStopping (std::string const& collision_name) : CollisionBase(collision_name) { using namespace amrex::literals; diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index a9e081d44b4..b3d39a9c581 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -8,6 +8,7 @@ #define WARPX_PARTICLES_COLLISION_BINARYCOLLISION_H_ #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" @@ -58,14 +59,14 @@ /** * \brief This class performs generic binary collisions. * - * \tparam CollisionFunctorType the type of the specific binary collision functor that acts on a + * \tparam CollisionFunctor the specific binary collision functor that acts on a * single cell - * \tparam CopyTransformFunctorType the type of the second functor used in the case of + * \tparam CopyTransformFunctor the second functor used in the case of * particle creation * */ -template +template class BinaryCollision final : public CollisionBase { @@ -74,7 +75,6 @@ class BinaryCollision final using ParticleTileType = WarpXParticleContainer::ParticleTileType; using ParticleTileDataType = ParticleTileType::ParticleTileDataType; using ParticleBins = amrex::DenseBins; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; public: @@ -92,18 +92,27 @@ public: WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); } + const CollisionType collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_isSameSpecies = (m_species_names[0] == m_species_names[1]); - m_binary_collision_functor = CollisionFunctorType(collision_name, mypc, m_isSameSpecies); + m_binary_collision_functor = CollisionFunctor(collision_name, mypc, m_isSameSpecies); const amrex::ParmParse pp_collision_name(collision_name); pp_collision_name.queryarr("product_species", m_product_species); + + // if DSMC the colliding species are also product species + // Therefore, we insert the colliding species at the beginning of `m_product_species` + if (collision_type == CollisionType::DSMC) { + m_product_species.insert( m_product_species.begin(), m_species_names.begin(), m_species_names.end() ); + } m_have_product_species = !m_product_species.empty(); - if ((std::is_same::value) & (m_have_product_species)) { + + if ((std::is_same::value) && (m_have_product_species)) { WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + " does not produce species. Thus, `product_species` should not be specified in the input script." ); } - m_copy_transform_functor = CopyTransformFunctorType(collision_name, mypc); + m_copy_transform_functor = CopyTransformFunctor(collision_name, mypc); } ~BinaryCollision () override = default; @@ -111,7 +120,6 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) = delete; BinaryCollision& operator= ( BinaryCollision&& ) = delete; @@ -157,15 +165,15 @@ public: amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, copy_species2.begin(), copy_species2.end(), device_copy_species2.begin()); amrex::Gpu::streamSynchronize(); - auto copy_species1_data = device_copy_species1.data(); - auto copy_species2_data = device_copy_species2.data(); + auto *copy_species1_data = device_copy_species1.data(); + auto *copy_species2_data = device_copy_species2.data(); #else auto *copy_species1_data = copy_species1.data(); auto *copy_species2_data = copy_species2.data(); #endif if (m_have_product_species){ species1.defineAllParticleTiles(); - species2.defineAllParticleTiles(); + if (!m_isSameSpecies) { species2.defineAllParticleTiles(); } } // Enable tiling @@ -190,7 +198,7 @@ public: auto wt = static_cast(amrex::second()); doCollisionsWithinTile( dt, lev, mfi, species1, species2, product_species_vector, - copy_species1_data, copy_species2_data); + copy_species1_data, copy_species2_data); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { @@ -199,6 +207,14 @@ public: amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } + + if (m_have_product_species) { + // The fact that there are product species indicates that particles of + // the colliding species (`species1` and `species2`) may be removed + // (i.e., marked as invalid) in the process of creating new product particles. + species1.deleteInvalidParticles(); + if (!m_isSameSpecies) { species2.deleteInvalidParticles(); } + } } } @@ -224,7 +240,7 @@ public: using namespace ParticleUtils; using namespace amrex::literals; - CollisionFunctorType binary_collision_functor = m_binary_collision_functor; + const auto& binary_collision_functor = m_binary_collision_functor.executor(); const bool have_product_species = m_have_product_species; // Store product species data in vectors @@ -281,7 +297,6 @@ public: auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); #endif - /* The following calculations are only required when creating product particles */ @@ -308,6 +323,25 @@ public: p_n_pairs_in_each_cell, pair_offsets.data()); index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + amrex::Gpu::DeviceVector n_ind_pairs_in_each_cell(n_cells+1); + index_type* AMREX_RESTRICT p_n_ind_pairs_in_each_cell = n_ind_pairs_in_each_cell.dataPtr(); + + amrex::ParallelFor( n_cells+1, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell = (i_cell < n_cells)? cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]: 0; + // number of independent collisions in each cell + p_n_ind_pairs_in_each_cell[i_cell] = n_part_in_cell/2; + } + ); + + // start indices of independent collisions. + amrex::Gpu::DeviceVector coll_offsets(n_cells+1); + // number of total independent collision pairs + const auto n_independent_pairs = (int) amrex::Scan::ExclusiveSum(n_cells+1, + p_n_ind_pairs_in_each_cell, coll_offsets.data(), amrex::Scan::RetSum{true}); + index_type* AMREX_RESTRICT p_coll_offsets = coll_offsets.dataPtr(); + // mask: equal to 1 if particle creation occurs for a given pair, 0 otherwise amrex::Gpu::DeviceVector mask(n_total_pairs); index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); @@ -326,7 +360,6 @@ public: End of calculations only required when creating product particles */ - // Loop over cells amrex::ParallelForRNG( n_cells, [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept @@ -337,16 +370,44 @@ public: index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; - // Same but for the pairs - index_type const cell_start_pair = have_product_species? - p_pair_offsets[i_cell] : 0; - // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( indices_1, cell_start_1, cell_half_1, engine ); + } + ); + + // Loop over independent particle pairs + // To speed up binary collisions on GPU, we try to expose as much parallelism + // as possible (while avoiding race conditions): Instead of looping with one GPU + // thread per cell, we loop with one GPU thread per "independent pairs" (i.e. pairs + // that do not touch the same macroparticles, so that there is no race condition), + // where the number of independent pairs is determined by the lower number of + // macroparticles of either species, within each cell. + amrex::ParallelForRNG( n_independent_pairs, + [=] AMREX_GPU_DEVICE (int i_coll, amrex::RandomEngine const& engine) noexcept + { + // to avoid type mismatch errors + auto ui_coll = (index_type)i_coll; + + // Use a bisection algorithm to find the index of the cell in which this pair is located + const int i_cell = amrex::bisect( p_coll_offsets, 0, n_cells, ui_coll ); + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; + + // collision number of the cell + const index_type coll_idx = ui_coll - p_coll_offsets[i_cell]; + + // Same but for the pairs + index_type const cell_start_pair = have_product_species? + p_pair_offsets[i_cell] : 0; + #if defined WARPX_DIM_RZ const int ri = (i_cell - i_cell%nz) / nz; auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; @@ -359,16 +420,16 @@ public: cell_half_1, cell_stop_1, indices_1, indices_1, soa_1, soa_1, get_position_1, get_position_1, - q1, q1, m1, m1, dt, dV, + q1, q1, m1, m1, dt, dV, coll_idx, cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, engine ); + p_pair_reaction_weight, engine); } ); // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, + ptile_1, ptile_1, product_species_vector, tile_products_data, m1, m1, @@ -427,7 +488,6 @@ public: auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); #endif - /* The following calculations are only required when creating product particles */ @@ -460,6 +520,32 @@ public: p_n_pairs_in_each_cell, pair_offsets.data()); index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + amrex::Gpu::DeviceVector n_ind_pairs_in_each_cell(n_cells+1); + index_type* AMREX_RESTRICT p_n_ind_pairs_in_each_cell = n_ind_pairs_in_each_cell.dataPtr(); + + amrex::ParallelFor( n_cells+1, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + if (i_cell < n_cells) + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + p_n_ind_pairs_in_each_cell[i_cell] = amrex::min(n_part_in_cell_1, n_part_in_cell_2); + } + else + { + p_n_ind_pairs_in_each_cell[i_cell] = 0; + } + } + ); + + // start indices of independent collisions. + amrex::Gpu::DeviceVector coll_offsets(n_cells+1); + // number of total independent collision pairs + const auto n_independent_pairs = (int) amrex::Scan::ExclusiveSum(n_cells+1, + p_n_ind_pairs_in_each_cell, coll_offsets.data(), amrex::Scan::RetSum{true}); + index_type* AMREX_RESTRICT p_coll_offsets = coll_offsets.dataPtr(); + // mask: equal to 1 if particle creation occurs for a given pair, 0 otherwise amrex::Gpu::DeviceVector mask(n_total_pairs); index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); @@ -490,9 +576,6 @@ public: // Same for species 2 index_type const cell_start_2 = cell_offsets_2[i_cell]; index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; - // Same but for the pairs - index_type const cell_start_pair = have_product_species? - p_pair_offsets[i_cell] : 0; // ux from species1 can be accessed like this: // ux_1[ indices_1[i] ], where i is between @@ -505,6 +588,44 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); + } + ); + + // Loop over independent particle pairs + // To speed up binary collisions on GPU, we try to expose as much parallelism + // as possible (while avoiding race conditions): Instead of looping with one GPU + // thread per cell, we loop with one GPU thread per "independent pairs" (i.e. pairs + // that do not touch the same macroparticles, so that there is no race condition), + // where the number of independent pairs is determined by the lower number of + // macroparticles of either species, within each cell. + amrex::ParallelForRNG( n_independent_pairs, + [=] AMREX_GPU_DEVICE (int i_coll, amrex::RandomEngine const& engine) noexcept + { + // to avoid type mismatch errors + auto ui_coll = (index_type)i_coll; + + // Use a bisection algorithm to find the index of the cell in which this pair is located + const int i_cell = amrex::bisect( p_coll_offsets, 0, n_cells, ui_coll ); + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + + // collision number of the cell + const index_type coll_idx = ui_coll - p_coll_offsets[i_cell]; + + // Same but for the pairs + index_type const cell_start_pair = have_product_species? + p_pair_offsets[i_cell]: 0; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + #if defined WARPX_DIM_RZ const int ri = (i_cell - i_cell%nz) / nz; auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; @@ -516,17 +637,16 @@ public: cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, indices_1, indices_2, soa_1, soa_2, get_position_1, get_position_2, - q1, q2, m1, m2, dt, dV, + q1, q2, m1, m2, dt, dV, coll_idx, cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, engine ); + p_pair_reaction_weight, engine); } ); - // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, + ptile_1, ptile_2, product_species_vector, tile_products_data, m1, m2, @@ -550,9 +670,9 @@ private: bool m_have_product_species; amrex::Vector m_product_species; // functor that performs collisions within a cell - CollisionFunctorType m_binary_collision_functor; + CollisionFunctor m_binary_collision_functor; // functor that creates new particles and initializes their parameters - CopyTransformFunctorType m_copy_transform_functor; + CopyTransformFunctor m_copy_transform_functor; }; diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index b8a390ddb93..2e24a93a35e 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef BINARY_COLLISION_UTILS_H_ -#define BINARY_COLLISION_UTILS_H_ +#ifndef WARPX_BINARY_COLLISION_UTILS_H_ +#define WARPX_BINARY_COLLISION_UTILS_H_ #include @@ -19,6 +19,8 @@ enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, DeuteriumHeliumToProtonHeliumFusion, ProtonBoronToAlphasFusion, + DSMC, + PairwiseCoulomb, Undefined }; enum struct NuclearFusionType { @@ -31,10 +33,10 @@ enum struct NuclearFusionType { namespace BinaryCollisionUtils{ - NuclearFusionType get_nuclear_fusion_type (std::string collision_name, + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * mypc); - CollisionType get_collision_type (std::string collision_name, + CollisionType get_collision_type (const std::string& collision_name, MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); @@ -60,60 +62,66 @@ namespace BinaryCollisionUtils{ using namespace amrex::literals; using namespace amrex::Math; - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); constexpr double c_sq = PhysConst::c * PhysConst::c; constexpr double inv_csq = 1.0 / c_sq; - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; + // Cast input parameters to double before computing collision properties + // This is needed to avoid errors when using single-precision particles + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const auto u1x_dbl = static_cast(u1x); + const auto u1y_dbl = static_cast(u1y); + const auto u1z_dbl = static_cast(u1z); + const auto u2x_dbl = static_cast(u2x); + const auto u2y_dbl = static_cast(u2y); + const auto u2z_dbl = static_cast(u2z); + + const double m1_sq = m1_dbl*m1_dbl; + const double m2_sq = m2_dbl*m2_dbl; // Compute Lorentz factor gamma in the lab frame - const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + const double g1 = std::sqrt( 1.0 + (u1x_dbl*u1x_dbl + u1y_dbl*u1y_dbl + u1z_dbl*u1z_dbl)*inv_csq ); + const double g2 = std::sqrt( 1.0 + (u2x_dbl*u2x_dbl + u2y_dbl*u2y_dbl + u2z_dbl*u2z_dbl)*inv_csq ); // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; + const double p1x = u1x_dbl * m1_dbl; + const double p1y = u1y_dbl * m1_dbl; + const double p1z = u1z_dbl * m1_dbl; + const double p2x = u2x_dbl * m2_dbl; + const double p2y = u2y_dbl * m2_dbl; + const double p2z = u2z_dbl * m2_dbl; + // Square norm of the total (sum between the two particles) momenta in the lab frame - const auto p_total_sq = static_cast( - powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) - ); + const double p_total_sq = powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z); // Total energy in the lab frame // Note the use of `double` for energy since this calculation is // prone to error with single precision. - const auto m1_dbl = static_cast(m1); - const auto m2_dbl = static_cast(m2); - const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + const double E_lab = (m1_dbl*g1 + m2_dbl*g2) * c_sq; // Total energy squared in the center of mass frame, calculated using the Lorentz invariance // of the four-momentum norm const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; // Kinetic energy in the center of mass frame const double E_star = std::sqrt(E_star_sq); + + // Cast back to chosen precision for output E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); // Square of the norm of the momentum of one of the particles in the center of mass frame // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle // The expression below is specifically written in a form that avoids returning // small negative numbers due to machine precision errors, for low-energy particles - const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); - const auto p_star_sq = static_cast( - m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) - ); + const double E_ratio = E_star/((m1_dbl + m2_dbl)*c_sq); + const double p_star_sq = m1_dbl*m2_dbl*c_sq * ( powi<2>(E_ratio) - 1.0 ) + + powi<2>(m1_dbl - m2_dbl)*c_sq/4.0 * powi<2>( E_ratio - 1.0/E_ratio); // Lorentz factors in the center of mass frame - const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); - const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + const double g1_star = std::sqrt(1.0 + p_star_sq / (m1_sq*c_sq)); + const double g2_star = std::sqrt(1.0 + p_star_sq / (m2_sq*c_sq)); - // relative velocity in the center of mass frame - v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + // relative velocity in the center of mass frame, cast back to chosen precision + v_rel_COM = static_cast(std::sqrt(p_star_sq) * (1.0/(m1_dbl*g1_star) + 1.0/(m2_dbl*g2_star))); // Cross sections and relative velocity are computed in the center of mass frame. // On the other hand, the particle densities (weight over volume) in the lab frame are used. @@ -122,8 +130,36 @@ namespace BinaryCollisionUtils{ // COM frame and the Lorentz factors in the lab frame (see // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor // is calculated here. - lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + lab_to_COM_lorentz_factor = static_cast(g1_star*g2_star/(g1*g2)); + } + + /** + * \brief Subtract given weight from particle and set its ID to invalid + * if the weight reaches zero. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void remove_weight_from_colliding_particle ( + amrex::ParticleReal& weight, uint64_t& idcpu, + const amrex::ParticleReal reaction_weight ) + { + // Remove weight from given particle + amrex::Gpu::Atomic::AddNoRet(&weight, -reaction_weight); + + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to invalid + if (weight <= std::numeric_limits::min()) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + } } } -#endif // BINARY_COLLISION_UTILS_H_ +#endif // WARPX_BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 430bac5e548..05756583554 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -17,123 +17,129 @@ namespace BinaryCollisionUtils{ - NuclearFusionType get_nuclear_fusion_type (const std::string collision_name, + CollisionType get_collision_type (const std::string& collision_name, + MultiParticleContainer const * const mypc) + { + const amrex::ParmParse pp_collision_name(collision_name); + // For legacy, pairwisecoulomb is the default + std::string type = "pairwisecoulomb"; + pp_collision_name.query("type", type); + if (type == "pairwisecoulomb") { + return CollisionType::PairwiseCoulomb; + } + else if (type == "nuclearfusion") { + const NuclearFusionType fusion_type = get_nuclear_fusion_type(collision_name, mypc); + return nuclear_fusion_type_to_collision_type(fusion_type); + } + else if (type == "dsmc") { + return CollisionType::DSMC; + } + return CollisionType::Undefined; + } + + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); - amrex::Vector species_names; - pp_collision_name.getarr("species", species_names); - auto& species1 = mypc->GetParticleContainerFromName(species_names[0]); - auto& species2 = mypc->GetParticleContainerFromName(species_names[1]); - amrex::Vector product_species_name; - pp_collision_name.getarr("product_species", product_species_name); + { + const amrex::ParmParse pp_collision_name(collision_name); + amrex::Vector species_names; + pp_collision_name.getarr("species", species_names); + auto& species1 = mypc->GetParticleContainerFromName(species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(species_names[1]); + amrex::Vector product_species_name; + pp_collision_name.getarr("product_species", product_species_name); - if ((species1.AmIA() && species2.AmIA()) - || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-tritium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) - || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-tritium fusion must be of type neutron and helium4"); - return NuclearFusionType::DeuteriumTritiumToNeutronHelium; - } - else if (species1.AmIA() && species2.AmIA()) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-deuterium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA())){ - return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; - } else if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA())){ - return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; - } else { - WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); - } - } - else if ((species1.AmIA() && species2.AmIA()) + if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-tritium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (product_species1.AmIA() && product_species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-helium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) - || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); - return NuclearFusionType::DeuteriumHeliumToProtonHelium; + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-tritium fusion must be of type neutron and helium4"); + return NuclearFusionType::DeuteriumTritiumToNeutronHelium; + } + else if (species1.AmIA() && species2.AmIA()) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-deuterium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + if ( + (product_species1.AmIA() && product_species2.AmIA()) + ||(product_species1.AmIA() && product_species2.AmIA())){ + return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; + } else if ( + (product_species1.AmIA() && product_species2.AmIA()) + ||(product_species1.AmIA() && product_species2.AmIA())){ + return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; + } else { + WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); } - else if ((species1.AmIA() && species2.AmIA()) + } + else if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-helium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (product_species1.AmIA() && product_species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 1, - "ERROR: Proton-boron must contain exactly one product species"); - auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species.AmIA(), - "ERROR: Product species of proton-boron fusion must be of type helium4"); - return NuclearFusionType::ProtonBoronToAlphas; - } - WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + - species_names[0] + " of type " + species1.getSpeciesTypeName() + - " and species " + species_names[1] + " of type " + - species2.getSpeciesTypeName()); - return NuclearFusionType::Undefined; + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); + return NuclearFusionType::DeuteriumHeliumToProtonHelium; } - - CollisionType get_collision_type (const std::string collision_name, - MultiParticleContainer const * const mypc) + else if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) { - const amrex::ParmParse pp_collision_name(collision_name); - std::string type; - pp_collision_name.get("type", type); - if (type == "nuclearfusion") { - const NuclearFusionType fusion_type = get_nuclear_fusion_type(collision_name, mypc); - return nuclear_fusion_type_to_collision_type(fusion_type); - } - WARPX_ABORT_WITH_MESSAGE(type + " is not a valid type of collision that creates new particles"); - return CollisionType::Undefined; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 1, + "ERROR: Proton-boron must contain exactly one product species"); + auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species.AmIA(), + "ERROR: Product species of proton-boron fusion must be of type helium4"); + return NuclearFusionType::ProtonBoronToAlphas; } + WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + + species_names[0] + " of type " + species1.getSpeciesTypeName() + + " and species " + species_names[1] + " of type " + + species2.getSpeciesTypeName()); + return NuclearFusionType::Undefined; + } CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) - { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { - return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { - return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { - return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { - return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - } - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - return CollisionType::ProtonBoronToAlphasFusion; - } - WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); - return CollisionType::Undefined; + { + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { + return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { + return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { + return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { + return CollisionType::DeuteriumHeliumToProtonHeliumFusion; + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { + return CollisionType::ProtonBoronToAlphasFusion; } + WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); + return CollisionType::Undefined; + } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index d7403eeacf9..e407ae1603b 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -37,6 +37,7 @@ * @param[in] dV is the volume of the corresponding cell. * @param[in] engine the random number generator state & factory * @param[in] isSameSpecies whether this is an intra-species collision process + * @param[in] coll_idx is the collision index offset. */ template @@ -52,10 +53,12 @@ void ElasticCollisionPerez ( T_PR const T1, T_PR const T2, T_R const dt, T_PR const L, T_R const dV, amrex::RandomEngine const& engine, - bool const isSameSpecies) + bool const isSameSpecies, T_index coll_idx) { const T_index NI1 = I1e - I1s; const T_index NI2 = I2e - I2s; + const T_index max_N = amrex::max(NI1,NI2); + const T_index min_N = amrex::max(NI1,NI2); T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; @@ -95,7 +98,7 @@ void ElasticCollisionPerez ( n1 = n1 / dV; n2 = n2 / dV; { T_index i1 = I1s; T_index i2 = I2s; - for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) + for (T_index k = 0; k < max_N; ++k) { n12 += amrex::min( w1[ I1[i1] ], w2[ I2[i2] ] ); ++i1; if ( i1 == I1e ) { i1 = I1s; } @@ -126,10 +129,13 @@ void ElasticCollisionPerez ( // call UpdateMomentumPerezElastic() { - T_index i1 = I1s; T_index i2 = I2s; - for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) - { + T_index i1 = I1s + coll_idx; + T_index i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (T_index k = coll_idx; k < max_N; k += min_N) + { #if (defined WARPX_DIM_RZ) /* In RZ geometry, macroparticles can collide with other macroparticles * in the same *cylindrical* cell. For this reason, collisions between macroparticles @@ -159,8 +165,12 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); #endif - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index cfdc36d3c50..26782c2e42a 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PAIRWISE_COULOMB_COLLISION_FUNC_H_ -#define PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#ifndef WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#define WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ #include "ElasticCollisionPerez.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -46,7 +46,7 @@ public: * @param[in] mypc the particle container (unused) * @param[in] isSameSpecies true if this is an intra-species colission */ - PairWiseCoulombCollisionFunc (const std::string collision_name, + PairWiseCoulombCollisionFunc (const std::string& collision_name, [[maybe_unused]] MultiParticleContainer const * const mypc, const bool isSameSpecies): m_isSameSpecies{isSameSpecies} @@ -58,37 +58,42 @@ public: utils::parser::queryWithParser( pp_collision_name, "CoulombLog", CoulombLog); m_CoulombLog = CoulombLog; + + m_exe.m_CoulombLog = m_CoulombLog; + m_exe.m_isSameSpecies = m_isSameSpecies; } - /** - * \brief operator() of the PairWiseCoulombCollisionFunc class. Performs Coulomb collisions - * at the cell level by calling ElasticCollisionPerez. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. - * @param[in] q1,q2 are charges. - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] engine the random engine. - */ - AMREX_GPU_HOST_DEVICE AMREX_INLINE - void operator() ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, - amrex::ParticleReal const q1, amrex::ParticleReal const q2, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const /*cell_start_pair*/, index_type* /*p_mask*/, - index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, - amrex::ParticleReal* /*p_pair_reaction_weight*/, - amrex::RandomEngine const& engine) const + struct Executor { + /** + * \brief Executor of the PairWiseCoulombCollisionFunc class. Performs Coulomb collisions + * at the cell level by calling ElasticCollisionPerez. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. + * @param[in] q1,q2 are charges. + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const q1, amrex::ParticleReal const q2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, + index_type const /*cell_start_pair*/, index_type* /*p_mask*/, + index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, + amrex::ParticleReal* /*p_pair_reaction_weight*/, + amrex::RandomEngine const& engine) const { using namespace amrex::literals; @@ -96,12 +101,20 @@ public: I1s, I1e, I2s, I2e, I1, I2, soa_1, soa_2, q1, q2, m1, m2, -1.0_prt, -1.0_prt, - dt, m_CoulombLog, dV, engine, m_isSameSpecies); + dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); } + amrex::ParticleReal m_CoulombLog; + bool m_isSameSpecies; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + private: amrex::ParticleReal m_CoulombLog; bool m_isSameSpecies; + + Executor m_exe; }; -#endif // PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#endif // WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt index 3575d53ad29..60da82ee48f 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - DSMC.cpp + DSMCFunc.cpp + SplitAndScatterFunc.cpp ) endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index 64f05650d1f..46b228b049e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef COLLISION_FILTER_FUNC_H_ -#define COLLISION_FILTER_FUNC_H_ +#ifndef WARPX_COLLISION_FILTER_FUNC_H_ +#define WARPX_COLLISION_FILTER_FUNC_H_ #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" @@ -18,59 +18,69 @@ * \brief This function determines whether a collision occurs for a given * pair of particles. * - * @param[in] u1x,u1y,u1z momenta of the first colliding particle - * @param[in] u2x,u2y,u2z momenta of the second colliding particle - * @param[in] m1,m2 masses - * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] u1x,u1y,u1z momenta of the first colliding particle. + * @param[in] u2x,u2y,u2z momenta of the second colliding particle. + * @param[in] m1,m2 masses. + * @param[in] w1,w2 effective weight of the colliding particles. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. - * @param[in] pair_index is the index of the colliding pair + * @param[in] pair_index is the index of the colliding pair. * @param[out] p_mask is a mask that will be set to a non-zero integer if a * collision occurs. The integer encodes the scattering process. - * @param[out] p_pair_reaction_weight stores the weight of the product particles - * @param[in] process_count number of scattering processes to consider - * @param[in] scattering processes an array of scattering processes included for consideration + * @param[out] p_pair_reaction_weight stores the weight of the product particles. + * @param[in] multiplier factor by which the collision probability is increased to + * account for all other possible binary collision partners. + * @param[in] process_count number of scattering processes to consider. + * @param[in] scattering processes an array of scattering processes included for consideration. * @param[in] engine the random engine. */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, - const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, - const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, - const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, - amrex::ParticleReal w1, amrex::ParticleReal w2, - const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, +void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleReal u1y, + const amrex::ParticleReal u1z, const amrex::ParticleReal u2x, + const amrex::ParticleReal u2y, const amrex::ParticleReal u2z, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const amrex::ParticleReal w1, const amrex::ParticleReal w2, + const amrex::Real dt, const amrex::ParticleReal dV, const int pair_index, index_type* AMREX_RESTRICT p_mask, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - const int& multiplier_ratio, - int const process_count, - ScatteringProcess::Executor* scattering_processes, + const int multiplier, + const int process_count, + const ScatteringProcess::Executor* scattering_processes, const amrex::RandomEngine& engine) { - using namespace amrex::literals; - amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; - const amrex::ParticleReal w_min = amrex::min(w1, w2); - const amrex::ParticleReal w_max = amrex::max(w1, w2); - BinaryCollisionUtils::get_collision_parameters( u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, E_coll, v_coll, lab_to_COM_factor); - // convert E_coll to eV + using namespace amrex::literals; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + // convert E_coll from Joule to eV E_coll /= PhysConst::q_e; - amrex::ParticleReal sigma_tot = 0._prt; + // Evaluate the cross-section for each scattering process to determine + // the total collision probability. + AMREX_ASSERT_WITH_MESSAGE( + (process_count < 4), "Too many scattering processes in DSMC routine." + ); + int coll_type[4] = {0, 0, 0, 0}; + amrex::ParticleReal sigma_sums[4] = {0._prt, 0._prt, 0._prt, 0._prt}; for (int ii = 0; ii < process_count; ii++) { - auto const& scattering_process = *(scattering_processes + ii); - sigma_tot += scattering_process.getCrossSection(E_coll); + auto const& scattering_process = scattering_processes[ii]; + coll_type[ii] = int(scattering_process.m_type); + const amrex::ParticleReal sigma = scattering_process.getCrossSection(E_coll); + sigma_sums[ii] = sigma + ((ii == 0) ? 0._prt : sigma_sums[ii-1]); } + const auto sigma_tot = sigma_sums[process_count-1]; // calculate total collision probability const amrex::ParticleReal exponent = ( - lab_to_COM_factor * multiplier_ratio * w_max - * sigma_tot * v_coll * dt / dV + lab_to_COM_factor * multiplier * w_max * sigma_tot * v_coll * dt / dV ); // Compute actual collision probability that is always between zero and one @@ -84,13 +94,10 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR if (amrex::Random(engine) < probability) { const amrex::ParticleReal random_number = amrex::Random(engine); - amrex::ParticleReal sigma = 0._prt; for (int ii = 0; ii < process_count; ii++) { - auto const& scattering_process = *(scattering_processes + ii); - sigma += scattering_process.getCrossSection(E_coll); - if (random_number <= sigma / sigma_tot) + if (random_number <= sigma_sums[ii] / sigma_tot) { - p_mask[pair_index] = int(scattering_process.m_type); + p_mask[pair_index] = coll_type[ii]; p_pair_reaction_weight[pair_index] = w_min; break; } @@ -102,121 +109,4 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR } } -/** - * \brief Function that determines if a collision occurs and if so, what - * type. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in] ptd_1,ptd_2 contain the particle data of the two species - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] cell_start_pair is the start index of the pairs in that cell. - * @param[out] p_mask is a mask that will be set to a non-zero integer if a - * collision occurs. The integer encodes the scattering process. - * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the - * particles of a given pair. They are only needed here to store information that will be used - * later on when actually creating the product particles. - * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only - * needed here to store information that will be used later on when actually creating the - * product particles. - * @param[in] process_count number of scattering processes to consider - * @param[in] scattering processes an array of scattering processes included for consideration - * @param[in] engine the random engine. - */ -template -AMREX_GPU_HOST_DEVICE AMREX_INLINE -void CollisionFilter ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - PData ptd_1, PData ptd_2, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, - index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - int const process_count, - ScatteringProcess::Executor* scattering_processes, - amrex::RandomEngine const& engine) -{ - - amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; - amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; - amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; - - amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; - amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; - amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; - - // Number of macroparticles of each species - const int NI1 = I1e - I1s; - const int NI2 = I2e - I2s; - const int max_N = amrex::max(NI1,NI2); - - int i1 = I1s; - int i2 = I2s; - int pair_index = cell_start_pair; - - // Because the number of particles of each species is not always equal (NI1 != NI2 - // in general), some macroparticles will be paired with multiple macroparticles of the - // other species and we need to decrease their weight accordingly. - // c1 corresponds to the minimum number of times a particle of species 1 will be paired - // with a particle of species 2. Same for c2. - const int c1 = amrex::max(NI2/NI1,1); - const int c2 = amrex::max(NI1/NI2,1); - -#if (defined WARPX_DIM_RZ) - amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; - amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; -#endif - - for (int k = 0; k < max_N; ++k) - { - // c1k : how many times the current particle of species 1 is paired with a particle - // of species 2. Same for c2k. - const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; - const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; - -#if (defined WARPX_DIM_RZ) - /* In RZ geometry, macroparticles can collide with other macroparticles - * in the same *cylindrical* cell. For this reason, collisions between macroparticles - * are actually not local in space. In this case, the underlying assumption is that - * particles within the same cylindrical cell represent a cylindrically-symmetry - * momentum distribution function. Therefore, here, we temporarily rotate the - * momentum of one of the macroparticles in agreement with this cylindrical symmetry. - * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; - * there is a corresponding assert statement at initialization.) */ - amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; - amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); - u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); -#endif - - CollisionPairFilter( - u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], - u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], - m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, - dt, dV, pair_index, p_mask, p_pair_reaction_weight, - max_N, process_count, scattering_processes, engine); - -#if (defined WARPX_DIM_RZ) - amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); - u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); -#endif - - p_pair_indices_1[pair_index] = I1[i1]; - p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } - ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } - ++pair_index; - } -} - -#endif // COLLISION_FILTER_FUNC_H_ +#endif // WARPX_COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H deleted file mode 100644 index ab01eba2c81..00000000000 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright 2023 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: Roelof Groenewald (TAE Technologies) - * - * License: BSD-3-Clause-LBNL - */ -#ifndef DSMC_H_ -#define DSMC_H_ - -#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" -#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" -#include "Particles/Collision/CollisionBase.H" -#include "Particles/Collision/ScatteringProcess.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleCreation/SmartCopy.H" -#include "Particles/ParticleCreation/SmartUtils.H" -#include "Particles/WarpXParticleContainer.H" -#include "Utils/Parser/ParserUtils.H" -#include "Utils/ParticleUtils.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include -#include - - -/** - * \brief This class performs DSMC (direct simulation Monte Carlo) collisions - * within a cell. Particles are paired up and for each pair a stochastic process - * determines whether a collision occurs. The algorithm is similar to the one - * used for binary Coulomb collisions and the nuclear fusion module. - */ -class DSMC final - : public CollisionBase -{ - // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using index_type = ParticleBins::index_type; - -public: - - /** - * \brief Constructor of the DSMC class - * - * @param[in] collision_name the name of the collision - */ - DSMC (std::string collision_name); - - /** Perform the collisions - * - * @param cur_time Current time - * @param dt Time step size - * @param mypc Container of species involved - * - */ - void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; - - /** Perform all binary collisions within a tile - * - * \param[in] lev the mesh-refinement level - * \param[in] mfi iterator for multifab - * \param species_1 first species container - * \param species_2 second species container - * \param copy_species1 SmartCopy for species_1 - * \param copy_species2 SmartCopy for species_2 - * - */ - void doCollisionsWithinTile ( - amrex::Real dt, int lev, amrex::MFIter const& mfi, - WarpXParticleContainer& species_1, - WarpXParticleContainer& species_2, - SmartCopy& copy_species1, - SmartCopy& copy_species2 ); - -private: - amrex::Vector m_scattering_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; -}; - -#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp deleted file mode 100644 index 92ae3ddf257..00000000000 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp +++ /dev/null @@ -1,300 +0,0 @@ -/* Copyright 2023 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: Roelof Groenewald (TAE Technologies) - * - * License: BSD-3-Clause-LBNL - */ -#include "CollisionFilterFunc.H" -#include "DSMC.H" -#include "SplitAndScatterFunc.H" - - -DSMC::DSMC (const std::string collision_name) - : CollisionBase(collision_name) -{ - using namespace amrex::literals; - const amrex::ParmParse pp_collision_name(collision_name); - -#if defined WARPX_DIM_RZ - amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); -#endif - - if(m_species_names.size() != 2) - { - amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); - } - - // query for a list of collision processes - // these could be elastic, excitation, charge_exchange, back, etc. - amrex::Vector scattering_process_names; - pp_collision_name.queryarr("scattering_processes", scattering_process_names); - - // create a vector of ScatteringProcess objects from each scattering - // process name - for (const auto& scattering_process : scattering_process_names) { - const std::string kw_cross_section = scattering_process + "_cross_section"; - std::string cross_section_file; - pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); - - // if the scattering process is excitation or ionization get the - // energy associated with that process - amrex::ParticleReal energy = 0._prt; - if (scattering_process.find("excitation") != std::string::npos || - scattering_process.find("ionization") != std::string::npos) { - const std::string kw_energy = scattering_process + "_energy"; - utils::parser::getWithParser( - pp_collision_name, kw_energy.c_str(), energy); - } - - ScatteringProcess process(scattering_process, cross_section_file, energy); - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, - "Cannot add an unknown scattering process type"); - - m_scattering_processes.push_back(std::move(process)); - } - -#ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - for (auto const& p : m_scattering_processes) { - h_scattering_processes_exe.push_back(p.executor()); - } - m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), - h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); - amrex::Gpu::streamSynchronize(); -#else - for (auto const& p : m_scattering_processes) { - m_scattering_processes_exe.push_back(p.executor()); - } -#endif -} - -void -DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) -{ - WARPX_PROFILE("DSMC::doCollisions()"); - - auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); - auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); - - // SmartCopy objects are created that will facilitate the particle splitting - // operation involved in DSMC collisions between particles with arbitrary - // weights. - const auto copy_factory_species1 = SmartCopyFactory{species1, species1}; - const auto copy_factory_species2 = SmartCopyFactory{species2, species2}; - auto copy_species1 = copy_factory_species1.getSmartCopy(); - auto copy_species2 = copy_factory_species2.getSmartCopy(); - - species1.defineAllParticleTiles(); - species2.defineAllParticleTiles(); - - // Enable tiling - amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } - - // Loop over refinement levels - for (int lev = 0; lev <= species1.finestLevel(); ++lev){ - - amrex::LayoutData* cost = WarpX::getCosts(lev); - - // Loop over all grids/tiles at this level -#ifdef AMREX_USE_OMP - info.SetDynamic(true); -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - } - auto wt = static_cast(amrex::second()); - - doCollisionsWithinTile(dt, lev, mfi, species1, species2, - copy_species1, copy_species2); - - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - wt = static_cast(amrex::second()) - wt; - amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); - } - } - - // Call redistribute to remove particles with negative ids - species1.Redistribute(lev, lev, 0, true, true); - species2.Redistribute(lev, lev, 0, true, true); - } -} - -void -DSMC::doCollisionsWithinTile( - amrex::Real dt, int const lev, amrex::MFIter const& mfi, - WarpXParticleContainer& species_1, - WarpXParticleContainer& species_2, - SmartCopy& copy_species1, - SmartCopy& copy_species2) -{ - using namespace ParticleUtils; - using namespace amrex::literals; - - // get collision processes - auto *scattering_processes = m_scattering_processes_exe.data(); - int const process_count = static_cast(m_scattering_processes_exe.size()); - - // Extract particles in the tile that `mfi` points to - ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); - ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); - - // Find the particles that are in each cell of this tile - ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); - ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); - - // Extract low-level data - int const n_cells = static_cast(bins_1.numBins()); - - // - Species 1 - index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); - index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); - const amrex::ParticleReal m1 = species_1.getMass(); - const auto ptd_1 = ptile_1.getParticleTileData(); - - // - Species 2 - index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); - index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); - const amrex::ParticleReal m2 = species_2.getMass(); - const auto ptd_2 = ptile_2.getParticleTileData(); - - amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); -#if defined WARPX_DIM_1D_Z - auto dV = geom.CellSize(0); -#elif defined WARPX_DIM_XZ - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined WARPX_DIM_RZ - amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box - const auto lo = lbound(cbx); - const auto hi = ubound(cbx); - const int nz = hi.y-lo.y+1; - auto dr = geom.CellSize(0); - auto dz = geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif - - // In the following we set up the "mask" used for creating new particles - // (from splitting events). There is a mask index for every collision - // pair. Below we find the size of the mask based on the greater of the - // number of each species' particle in each cell. - amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); - index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); - - // Compute how many pairs in each cell and store in n_pairs_in_each_cell array - // For different species, the number of pairs in a cell is the number of particles of - // the species that has the most particles in that cell - amrex::ParallelFor( n_cells, - [=] AMREX_GPU_DEVICE (int i_cell) noexcept - { - const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; - const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; - // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) - { - p_n_pairs_in_each_cell[i_cell] = 0; - } - else - { - p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); - } - } - ); - - // Start indices of the pairs in a cell. Will be used for particle creation - amrex::Gpu::DeviceVector pair_offsets(n_cells); - const index_type n_total_pairs = (n_cells == 0) ? 0: - amrex::Scan::ExclusiveSum(n_cells, - p_n_pairs_in_each_cell, pair_offsets.data()); - index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); - - // Now we create the mask. In the DSMC scheme the mask will serve two - // purposes, 1) record whether a given pair should collide and 2) record - // the scattering process that should occur. - amrex::Gpu::DeviceVector mask(n_total_pairs); - index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); - - // Will be filled with the index of the first particle of a given pair - amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); - index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); - // Will be filled with the index of the second particle of a given pair - amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); - index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); - - // How much weight should be given to the produced particle - based on the - // weight of the collision partner which is not split - amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = - pair_reaction_weight.dataPtr(); - - // Loop over cells - amrex::ParallelForRNG( n_cells, - [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept - { - // The particles from species1 that are in the cell `i_cell` are - // given by the `indices_1[cell_start_1:cell_stop_1]` - index_type const cell_start_1 = cell_offsets_1[i_cell]; - index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - // Same for species 2 - index_type const cell_start_2 = cell_offsets_2[i_cell]; - index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; - // Same but for the pairs - index_type const cell_start_pair = p_pair_offsets[i_cell]; - - // ux from species1 can be accessed like this: - // ux_1[ indices_1[i] ], where i is between - // cell_start_1 (inclusive) and cell_start_2 (exclusive) - - // Do not collide if one species is missing in the cell - if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) { return; } - - // shuffle - ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); - ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); -#if defined WARPX_DIM_RZ - const int ri = (i_cell - i_cell%nz) / nz; - auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; -#endif - // Call the function in order to perform collisions - // If there are product species, p_mask, p_pair_indices_1/2, and - // p_pair_reaction_weight are filled here - CollisionFilter( - cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, - indices_1, indices_2, - ptd_1, ptd_2, - m1, m2, dt, dV, - cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, - process_count, scattering_processes, engine ); - } - ); - - const auto num_p_tile1 = ptile_1.numParticles(); - const auto num_p_tile2 = ptile_2.numParticles(); - - // Create the new product particles and define their initial values - // num_added: how many particles of each product species have been created - const int num_added = splitScatteringParticles( - n_total_pairs, - ptile_1, ptile_2, - p_mask, - copy_species1, copy_species2, - m1, m2, - p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight); - - if (num_added > 0) { - setNewParticleIDs(ptile_1, num_p_tile1, num_added); - setNewParticleIDs(ptile_2, num_p_tile2, num_added); - } -} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H new file mode 100644 index 00000000000..f28f3fb984c --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -0,0 +1,204 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DSMC_FUNC_H_ +#define WARPX_DSMC_FUNC_H_ + +#include "CollisionFilterFunc.H" + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMCFunc +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + +public: + + /** + * \brief Constructor of the DSMCFunc class. + */ + DSMCFunc () = default; + + /** + * \brief Constructor of the DSMCFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + * @param[in] isSameSpecies whether the two colliding species are the same + */ + DSMCFunc ( const std::string& collision_name, + MultiParticleContainer const * mypc, + bool isSameSpecies ); + + struct Executor { + /** + * \brief Executor of the DSMCFunc class. Performs DSMC collisions at the cell level. + * Note that this function does not yet create the product particles, but + * instead fills an array p_mask that stores which pairs result in a collision event. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] soa_1,soa_2 contain the struct of array data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given + * pair. It is only needed here to store information that will be used later on when actually + * creating the product particles. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + amrex::RandomEngine const& engine) const + { + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const index_type NI1 = I1e - I1s; + const index_type NI2 = I2e - I2s; + const index_type max_N = amrex::max(NI1,NI2); + const index_type min_N = amrex::min(NI1,NI2); + + index_type pair_index = cell_start_pair + coll_idx; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 + const index_type c1 = amrex::max(NI2/NI1, index_type(1)); + const index_type c2 = amrex::max(NI1/NI2, index_type(1)); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; +#endif + index_type i1 = I1s + coll_idx; + index_type i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (index_type k = coll_idx; k < max_N; k += min_N) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const index_type c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const index_type c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) + */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, static_cast(pair_index), p_mask, + p_pair_reaction_weight, static_cast(max_N), + m_process_count, m_scattering_processes_data, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } + pair_index += min_N; + } + } + + int m_process_count; + ScatteringProcess::Executor* m_scattering_processes_data; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + + Executor m_exe; +}; + +#endif // WARPX_DSMC_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp new file mode 100644 index 00000000000..9ee676bf002 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -0,0 +1,79 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "DSMCFunc.H" +#include "Utils/TextMsg.H" + +/** + * \brief Constructor of the DSMCFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + * @param[in] isSameSpecies whether the two colliding species are the same + */ +DSMCFunc::DSMCFunc ( + const std::string& collision_name, + [[maybe_unused]] MultiParticleContainer const * const mypc, + [[maybe_unused]] const bool isSameSpecies ) +{ + using namespace amrex::literals; + + const amrex::ParmParse pp_collision_name(collision_name); + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + const std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + const std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + + const int process_count = static_cast(m_scattering_processes.size()); + + // Store ScatteringProcess::Executor(s). +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(process_count); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif + + // Link executor to appropriate ScatteringProcess executors + m_exe.m_scattering_processes_data = m_scattering_processes_exe.data(); + m_exe.m_process_count = process_count; +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package index b4cfab89c64..087044e63cd 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -1,3 +1,4 @@ -CEXE_sources += DSMC.cpp +CEXE_sources += DSMCFunc.cpp +CEXE_sources += SplitAndScatterFunc.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index f684b60da78..a0c6ca8c8d4 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -1,4 +1,4 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * @@ -6,176 +6,260 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SPLIT_AND_SCATTER_FUNC_H_ -#define SPLIT_AND_SCATTER_FUNC_H_ +#ifndef WARPX_SPLIT_AND_SCATTER_FUNC_H_ +#define WARPX_SPLIT_AND_SCATTER_FUNC_H_ +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" -#include "Particles/NamedComponentParticleContainer.H" - -#include +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/ParticleUtils.H" /** - * \brief Function that performs the particle scattering and injection due - * to binary collisions. - * - * \return num_added the number of particles added to each species. + * \brief This class defines an operator to create product particles from DSMC + * collisions and sets the particle properties (position, momentum, weight). */ -template ::value, int> foo = 0> -int splitScatteringParticles ( - const index_type& n_total_pairs, - Tile& ptile1, Tile& ptile2, - const Index* AMREX_RESTRICT mask, - CopyFunc&& copy1, CopyFunc&& copy2, - const amrex::ParticleReal m1, const amrex::ParticleReal m2, - const index_type* AMREX_RESTRICT p_pair_indices_1, - const index_type* AMREX_RESTRICT p_pair_indices_2, - const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +class SplitAndScatterFunc { - using namespace amrex; + // Define shortcuts for frequently-used type names + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - if (n_total_pairs == 0) { - return 0; - } +public: + /** + * \brief Default constructor of the SplitAndScatterFunc class. + */ + SplitAndScatterFunc () = default; + + /** + * \brief Constructor of the SplitAndScatterFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + */ + SplitAndScatterFunc (const std::string& collision_name, MultiParticleContainer const * mypc); - Gpu::DeviceVector offsets(n_total_pairs); - Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + /** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ + AMREX_INLINE + amrex::Vector operator() ( + const index_type& n_total_pairs, + ParticleTileType& ptile1, ParticleTileType& ptile2, + const amrex::Vector& pc_products, + ParticleTileType** AMREX_RESTRICT tile_products, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const amrex::Vector& /*products_mass*/, + const index_type* AMREX_RESTRICT mask, + const amrex::Vector& products_np, + const SmartCopy* AMREX_RESTRICT copy_species1, + const SmartCopy* AMREX_RESTRICT copy_species2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) const + { + using namespace amrex::literals; - // The following is used to calculate the appropriate offsets. Note that - // a standard cummulative sum is not appropriate since the mask is also - // used to specify the type of collision and can therefore have values >1 - auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, - [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, - [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, - amrex::Scan::Type::exclusive, amrex::Scan::retSum - ); + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } - const auto ptile1_index = ptile1.numParticles(); - const auto ptile2_index = ptile2.numParticles(); - ptile1.resize(ptile1_index + num_added); - ptile2.resize(ptile2_index + num_added); + amrex::Gpu::DeviceVector offsets(n_total_pairs); + index_type* AMREX_RESTRICT offsets_data = offsets.data(); + const index_type* AMREX_RESTRICT p_offsets = offsets.dataPtr(); - const auto ptile1_data = ptile1.getParticleTileData(); - const auto ptile2_data = ptile2.getParticleTileData(); + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const total = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (index_type i) -> index_type { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (index_type i, index_type s) { offsets_data[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); - ParallelForRNG(n_total_pairs, - [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept - { - if (mask[i]) + amrex::Vector num_added_vec(m_num_product_species); + for (int i = 0; i < m_num_product_species; i++) { - // First, make copies of the colliding particles - copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); - copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); - - // Now we adjust the properties of the original and child particles, - // starting with the parent particles - auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; - auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; - uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; + // How many particles of product species i are created. + const index_type num_added = total * m_num_products_host[i]; + num_added_vec[i] = static_cast(num_added); + tile_products[i]->resize(products_np[i] + num_added); + } + + const auto soa_1 = ptile1.getParticleTileData(); + const auto soa_2 = ptile2.getParticleTileData(); + + amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; + + // Create necessary GPU vectors, that will be used in the kernel below + amrex::Vector soa_products; + for (int i = 0; i < m_num_product_species; i++) + { + soa_products.push_back(tile_products[i]->getParticleTileData()); + } +#ifdef AMREX_USE_GPU + amrex::Gpu::DeviceVector device_soa_products(m_num_product_species); + amrex::Gpu::DeviceVector device_products_np(m_num_product_species); + + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, soa_products.begin(), + soa_products.end(), + device_soa_products.begin()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, products_np.begin(), + products_np.end(), + device_products_np.begin()); + + amrex::Gpu::streamSynchronize(); + SoaData_type* AMREX_RESTRICT soa_products_data = device_soa_products.data(); + const index_type* AMREX_RESTRICT products_np_data = device_products_np.data(); #else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); + SoaData_type* AMREX_RESTRICT soa_products_data = soa_products.data(); + const index_type* AMREX_RESTRICT products_np_data = products_np.data(); #endif - }; - - // Remove p_pair_reaction_weight[i] from the colliding particles' weights. - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to -1. - Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); - if (w1 <= 0._prt) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); - } - Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); - if (w2 <= 0._prt) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); - } + const int* AMREX_RESTRICT p_num_products_device = m_num_products_device.data(); + + amrex::ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept + { + if (mask[i]) + { + // for now we ignore the possibility of having actual reaction + // products - only duplicating (splitting) of the colliding + // particles is supported. + + const auto product1_index = products_np_data[0] + + (p_offsets[i]*p_num_products_device[0] + 0); + // Make a copy of the particle from species 1 + copy_species1[0](soa_products_data[0], soa_1, static_cast(p_pair_indices_1[i]), + static_cast(product1_index), engine); + // Set the weight of the new particles to p_pair_reaction_weight[i] + soa_products_data[0].m_rdata[PIdx::w][product1_index] = p_pair_reaction_weight[i]; + + const auto product2_index = products_np_data[1] + + (p_offsets[i]*p_num_products_device[1] + 0); + // Make a copy of the particle from species 2 + copy_species2[1](soa_products_data[1], soa_2, static_cast(p_pair_indices_2[i]), + static_cast(product2_index), engine); + // Set the weight of the new particles to p_pair_reaction_weight[i] + soa_products_data[1].m_rdata[PIdx::w][product2_index] = p_pair_reaction_weight[i]; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); + + // Set the child particle properties appropriately + auto& ux1 = soa_products_data[0].m_rdata[PIdx::ux][product1_index]; + auto& uy1 = soa_products_data[0].m_rdata[PIdx::uy][product1_index]; + auto& uz1 = soa_products_data[0].m_rdata[PIdx::uz][product1_index]; + auto& ux2 = soa_products_data[1].m_rdata[PIdx::ux][product2_index]; + auto& uy2 = soa_products_data[1].m_rdata[PIdx::uy][product2_index]; + auto& uz2 = soa_products_data[1].m_rdata[PIdx::uz][product2_index]; - // Set the child particle properties appropriately - ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; - ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; - - auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; - auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; - auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; - auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; - auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; - auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; - - // for simplicity (for now) we assume non-relativistic particles - // and simply calculate the center-of-momentum velocity from the - // rest masses - auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); - auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); - auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); - - // transform to COM frame - ux1 -= uCOM_x; - uy1 -= uCOM_y; - uz1 -= uCOM_z; - ux2 -= uCOM_x; - uy2 -= uCOM_y; - uz2 -= uCOM_z; - - if (mask[i] == int(ScatteringProcessType::ELASTIC)) { - // randomly rotate the velocity vector for the first particle - ParticleUtils::RandomizeVelocity( - ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine - ); - // set the second particles velocity so that the total momentum - // is zero - ux2 = -ux1 * m1 / m2; - uy2 = -uy1 * m1 / m2; - uz2 = -uz1 * m1 / m2; - } else if (mask[i] == int(ScatteringProcessType::BACK)) { - // reverse the velocity vectors of both particles - ux1 *= -1.0_prt; - uy1 *= -1.0_prt; - uz1 *= -1.0_prt; - ux2 *= -1.0_prt; - uy2 *= -1.0_prt; - uz2 *= -1.0_prt; - } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { - if (m1 == m2) { - auto const temp_ux = ux1; - auto const temp_uy = uy1; - auto const temp_uz = uz1; - ux1 = ux2; - uy1 = uy2; - uz1 = uz2; - ux2 = temp_ux; - uy2 = temp_uy; - uz2 = temp_uz; + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (std::abs(m1 - m2) < 1e-28) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + amrex::Abort("Uneven mass charge-exchange not implemented yet."); + } } else { - Abort("Uneven mass charge-exchange not implemented yet."); + amrex::Abort("Unknown scattering process."); } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; } - else { - Abort("Unknown scattering process."); - } - // transform back to labframe - ux1 += uCOM_x; - uy1 += uCOM_y; - uz1 += uCOM_z; - ux2 += uCOM_x; - uy2 += uCOM_y; - uz2 += uCOM_z; + }); + + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + const int start_index = int(products_np[i]); + const int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); } - }); - Gpu::synchronize(); - return static_cast(num_added); -} -#endif // SPLIT_AND_SCATTER_FUNC_H_ + amrex::Gpu::synchronize(); + return num_added_vec; + } + +private: + // How many different type of species the collision produces + int m_num_product_species; + // Vectors of size m_num_product_species storing how many particles of a given species are + // produced by a collision event. These vectors are duplicated (one version for host and one + // for device) which is necessary with GPUs but redundant on CPU. + amrex::Gpu::DeviceVector m_num_products_device; + amrex::Gpu::HostVector m_num_products_host; + CollisionType m_collision_type; +}; +#endif // WARPX_SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp new file mode 100644 index 00000000000..de8de9b505d --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp @@ -0,0 +1,43 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "SplitAndScatterFunc.H" + +SplitAndScatterFunc::SplitAndScatterFunc (const std::string& collision_name, + MultiParticleContainer const * const mypc): + m_collision_type{BinaryCollisionUtils::get_collision_type(collision_name, mypc)} +{ + const amrex::ParmParse pp_collision_name(collision_name); + + if (m_collision_type == CollisionType::DSMC) + { + // here we can add logic to deal with cases where products are created, + // for example with impact ionization + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); +#ifndef AMREX_USE_GPU + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); +#endif + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in SplitAndScatterFunc"); + } + +#ifdef AMREX_USE_GPU + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); +#endif +} diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H index 718d386d5ba..e408600480e 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef BOSCH_HALE_FUSION_CROSS_SECTION_H -#define BOSCH_HALE_FUSION_CROSS_SECTION_H +#ifndef WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H +#define WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Utils/WarpXConst.H" @@ -110,4 +110,4 @@ amrex::ParticleReal BoschHaleFusionCrossSection ( return millibarn_to_sqm * astrophysical_factor/E_keV * std::exp(-B_G/std::sqrt(E_keV)); } -#endif // BOSCH_HALE_FUSION_CROSS_SECTION_H +#endif // WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b2a2112ca68..b6d6fe171de 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef NUCLEAR_FUSION_FUNC_H_ -#define NUCLEAR_FUSION_FUNC_H_ +#ifndef WARPX_NUCLEAR_FUSION_FUNC_H_ +#define WARPX_NUCLEAR_FUSION_FUNC_H_ #include "SingleNuclearFusionEvent.H" @@ -56,11 +56,12 @@ public: * @param[in] mypc pointer to the MultiParticleContainer * @param[in] isSameSpecies whether the two colliding species are the same */ - NuclearFusionFunc (const std::string collision_name, MultiParticleContainer const * const mypc, + NuclearFusionFunc (const std::string& collision_name, MultiParticleContainer const * const mypc, const bool isSameSpecies): m_fusion_multiplier{amrex::ParticleReal{1.0}}, // default fusion multiplier m_probability_threshold{amrex::ParticleReal{0.02}}, // default fusion probability threshold m_probability_target_value{amrex::ParticleReal{0.002}}, // default fusion probability target_value + m_fusion_type{BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc)}, m_isSameSpecies{isSameSpecies} { @@ -68,8 +69,6 @@ public: WARPX_ABORT_WITH_MESSAGE("Nuclear fusion module does not currently work with single precision"); #endif - m_fusion_type = BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc); - const amrex::ParmParse pp_collision_name(collision_name); utils::parser::queryWithParser( pp_collision_name, "fusion_multiplier", m_fusion_multiplier); @@ -78,61 +77,68 @@ public: utils::parser::queryWithParser( pp_collision_name, "fusion_probability_target_value", m_probability_target_value); + + m_exe.m_fusion_multiplier = m_fusion_multiplier; + m_exe.m_probability_threshold = m_probability_threshold; + m_exe.m_probability_target_value = m_probability_target_value; + m_exe.m_fusion_type = m_fusion_type; + m_exe.m_isSameSpecies = m_isSameSpecies; } - /** - * \brief operator() of the NuclearFusionFunc class. Performs nuclear fusions at the cell level - * using the algorithm described in Higginson et al., Journal of Computational Physics 388, - * 439-453 (2019). Note that this function does not yet create the product particles, but - * instead fills an array p_mask that stores which collisions result in a fusion event. - * - * Also note that there are three main differences between this implementation and the - * algorithm described in Higginson's paper: - * - 1) The transformation from the lab frame to the center of mass frame is nonrelativistic - * in Higginson's paper. Here, we implement a relativistic generalization. - * - 2) The behaviour when the estimated fusion probability is greater than one is not - * specified in Higginson's paper. Here, we provide an implementation using two runtime - * dependent parameters (fusion probability threshold and fusion probability target value). See - * documentation for more details. - * - 3) Here, we divide the weight of a particle by the number of times it is paired with other - * particles. This was not explicitly specified in Higginson's paper. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in] soa_1,soa_2 contain the struct of array data of the two species - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] cell_start_pair is the start index of the pairs in that cell. - * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given - * pair. It is only needed here to store information that will be used later on when actually - * creating the product particles. - * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the - * particles of a given pair. They are only needed here to store information that will be used - * later on when actually creating the product particles. - * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only - * needed here to store information that will be used later on when actually creating the - * product particles. - * @param[in] engine the random engine. - */ - AMREX_GPU_HOST_DEVICE AMREX_INLINE - void operator() ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, - amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, - index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - amrex::RandomEngine const& engine) const + struct Executor { + /** + * \brief Executor of the NuclearFusionFunc class. Performs nuclear fusions at the cell level + * using the algorithm described in Higginson et al., Journal of Computational Physics 388, + * 439-453 (2019). Note that this function does not yet create the product particles, but + * instead fills an array p_mask that stores which collisions result in a fusion event. + * + * Also note that there are three main differences between this implementation and the + * algorithm described in Higginson's paper: + * - 1) The transformation from the lab frame to the center of mass frame is nonrelativistic + * in Higginson's paper. Here, we implement a relativistic generalization. + * - 2) The behaviour when the estimated fusion probability is greater than one is not + * specified in Higginson's paper. Here, we provide an implementation using two runtime + * dependent parameters (fusion probability threshold and fusion probability target value). See + * documentation for more details. + * - 3) Here, we divide the weight of a particle by the number of times it is paired with other + * particles. This was not explicitly specified in Higginson's paper. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] soa_1,soa_2 contain the struct of array data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given + * pair. It is only needed here to store information that will be used later on when actually + * creating the product particles. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + amrex::RandomEngine const& engine) const { - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; @@ -147,10 +153,9 @@ public: const index_type NI1 = I1e - I1s; const index_type NI2 = I2e - I2s; const index_type max_N = amrex::max(NI1,NI2); + const index_type min_N = amrex::min(NI1,NI2); - index_type i1 = I1s; - index_type i2 = I2s; - index_type pair_index = cell_start_pair; + index_type pair_index = cell_start_pair + coll_idx; // Because the number of particles of each species is not always equal (NI1 != NI2 // in general), some macroparticles will be paired with multiple macroparticles of the @@ -169,8 +174,11 @@ public: amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; #endif - - for (index_type k = 0; k < max_N; ++k) + index_type i1 = I1s + coll_idx; + index_type i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (index_type k = coll_idx; k < max_N; k += min_N) { // c1k : how many times the current particle of species 1 is paired with a particle // of species 2. Same for c2k. @@ -210,13 +218,25 @@ public: p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } - ++pair_index; + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } + pair_index += min_N; } - } + amrex::ParticleReal m_fusion_multiplier; + amrex::ParticleReal m_probability_threshold; + amrex::ParticleReal m_probability_target_value; + NuclearFusionType m_fusion_type; + bool m_isSameSpecies; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + private: // Factor used to increase the number of fusion reaction by decreasing the weight of the // produced particles @@ -231,6 +251,8 @@ private: amrex::ParticleReal m_probability_target_value; NuclearFusionType m_fusion_type; bool m_isSameSpecies; + + Executor m_exe; }; -#endif // NUCLEAR_FUSION_FUNC_H_ +#endif // WARPX_NUCLEAR_FUSION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index ec9a1989189..4f0d663388a 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PROTON_BORON_FUSION_CROSS_SECTION_H -#define PROTON_BORON_FUSION_CROSS_SECTION_H +#ifndef WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H +#define WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H #include "Utils/WarpXConst.H" @@ -16,18 +16,15 @@ #include /** - * \brief Computes the total proton-boron fusion cross section in the range 0 < E < 3.5 MeV using - * the analytical fits given in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). - * For the record, note that there is a typo in equation (1) of this paper: the total cross section - * should read S(E)/E*exp(-sqrt(E_G/E)) instead of S(E)/E*exp(sqrt(E_G/E)) (minus sign in the - * exponential). + * \brief Computes the total proton-boron fusion cross section in the range 0 < E < 9.76 MeV using + * the analytical fits given in A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023). * * @param[in] E_keV the kinetic energy of the proton-boron pair in its center of mass frame, in * keV. * @return The total cross section in barn. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleReal& E_keV) +amrex::ParticleReal ProtonBoronFusionCrossSectionTentori (const amrex::ParticleReal& E_keV) { using namespace amrex::literals; using namespace amrex::Math; @@ -54,13 +51,13 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe // Compute astrophysical factor, in MeV barn, using the fits constexpr auto E_lim1 = 400._prt; // Limits between the different fit regions - constexpr auto E_lim2 = 642._prt; + constexpr auto E_lim2 = 668._prt; amrex::ParticleReal astrophysical_factor; if (E_keV < E_lim1) { constexpr auto C0 = 197._prt; - constexpr auto C1 = 0.24_prt; - constexpr auto C2 = 2.31e-4_prt; + constexpr auto C1 = 0.269_prt; + constexpr auto C2 = 2.54e-4_prt; constexpr auto AL = 1.82e4_prt; constexpr auto EL = 148._prt; constexpr auto dEL_sq = 2.35_prt*2.35_prt; @@ -69,28 +66,28 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe } else if (E_keV < E_lim2) { - constexpr auto D0 = 330._prt; - constexpr auto D1 = 66.1_prt; - constexpr auto D2 = -20.3_prt; - constexpr auto D5 = -1.58_prt; + constexpr auto D0 = 346._prt; + constexpr auto D1 = 150._prt; + constexpr auto D2 = -59.9_prt; + constexpr auto D5 = -0.460_prt; const amrex::ParticleReal E_norm = (E_keV-400._prt) * 1.e-2_prt; astrophysical_factor = D0 + D1*E_norm + D2*powi<2>(E_norm) + D5*powi<5>(E_norm); } else { - constexpr auto A0 = 2.57e6_prt; - constexpr auto A1 = 5.67e5_prt; - constexpr auto A2 = 1.34e5_prt; - constexpr auto A3 = 5.68e5_prt; - constexpr auto E0 = 581.3_prt; - constexpr auto E1 = 1083._prt; - constexpr auto E2 = 2405._prt; - constexpr auto E3 = 3344._prt; - constexpr auto dE0_sq = 85.7_prt*85.7_prt; - constexpr auto dE1_sq = 234._prt*234._prt; - constexpr auto dE2_sq = 138._prt*138._prt; - constexpr auto dE3_sq = 309._prt*309._prt; - constexpr auto B = 4.38_prt; + constexpr auto A0 = 1.98e6_prt; + constexpr auto A1 = 3.89e6_prt; + constexpr auto A2 = 1.36e6_prt; + constexpr auto A3 = 3.71e6_prt; + constexpr auto E0 = 640.9_prt; + constexpr auto E1 = 1211._prt; + constexpr auto E2 = 2340._prt; + constexpr auto E3 = 3294._prt; + constexpr auto dE0_sq = 85.5_prt*85.5_prt; + constexpr auto dE1_sq = 414._prt*414._prt; + constexpr auto dE2_sq = 221._prt*221._prt; + constexpr auto dE3_sq = 351._prt*351._prt; + constexpr auto B = 0.381_prt; astrophysical_factor = A0 / ((E_keV-E0)*(E_keV-E0) + dE0_sq) + A1 / ((E_keV-E1)*(E_keV-E1) + dE1_sq) + A2 / ((E_keV-E2)*(E_keV-E2) + dE2_sq) + @@ -102,10 +99,12 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe } /** - * \brief Computes the total proton-boron fusion cross section in the range E > 3.5 MeV using a + * \brief Computes the total proton-boron fusion cross section in the range E > 9.76 MeV using a * simple power law fit of the data presented in Buck et al., Nuclear Physics A, 398(2), 189-202 - * (1983) (data can also be found in the EXFOR database). - * + * (1983) (data can also be found in the EXFOR database). Note: the fit in Buck et al. started + * from 3.5 MeV. The same exponent power has been used here however the cross_section_start_fit + * has been modified to ensure exact continuity with the fit used for lower energies. + * @param[in] E_keV the kinetic energy of the proton-boron pair in its center of mass frame, in * keV. * @return The total cross section in barn. @@ -115,9 +114,9 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionBuck (const amrex::ParticleReal { using namespace amrex::literals; - constexpr amrex::ParticleReal E_start_fit = 3500._prt; // Fit starts at 3.5 MeV + constexpr amrex::ParticleReal E_start_fit = 9760._prt; // Fit starts at 9.76 MeV // cross section at E = E_start_fit, in barn - constexpr amrex::ParticleReal cross_section_start_fit = 0.2168440845211521_prt; + constexpr amrex::ParticleReal cross_section_start_fit = 0.01277998_prt; constexpr amrex::ParticleReal slope_fit = -2.661840717596765_prt; // Compute fitted value @@ -125,12 +124,13 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionBuck (const amrex::ParticleReal } /** - * \brief Computes the total proton-boron fusion cross section. When E_kin_star < 3.5 MeV, we use - * the analytical fits given in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). When - * E_kin_star > 3.5 MeV, we use a simple power law fit of the data presented in Buck et al., + * \brief Computes the total proton-boron fusion cross section using the analytical fit described + * in A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023). Includes the Breit-Wigner term + * to reconstruct the 148 keV resonance, which is missing from the Sikora, Wellar dataset. When + * E_kin_star > 9.76 MeV, we use a simple power law fit of the data presented in Buck et al., * Nuclear Physics A, 398(2), 189-202 (1983). Both fits return the same value for - * E_kin_star = 3.5 MeV. - * + * E_kin_star = 9.76 MeV. + * @param[in] E_kin_star the kinetic energy of the proton-boron pair in its center of mass frame, * in SI units. * @return The total cross section in SI units (square meters). @@ -143,10 +143,10 @@ amrex::ParticleReal ProtonBoronFusionCrossSection (const amrex::ParticleReal& E_ // Fits use energy in keV constexpr amrex::ParticleReal joule_to_keV = 1.e-3_prt/PhysConst::q_e; const amrex::ParticleReal E_keV = E_kin_star*joule_to_keV; - constexpr amrex::ParticleReal E_threshold = 3500._prt; + constexpr amrex::ParticleReal E_threshold = 9760._prt; const amrex::ParticleReal cross_section_b = (E_keV <= E_threshold) ? - ProtonBoronFusionCrossSectionNevins(E_keV) : + ProtonBoronFusionCrossSectionTentori(E_keV) : ProtonBoronFusionCrossSectionBuck(E_keV); // Convert cross section to SI units: barn to square meter @@ -154,4 +154,4 @@ amrex::ParticleReal ProtonBoronFusionCrossSection (const amrex::ParticleReal& E_ return cross_section_b*barn_to_sqm; } -#endif // PROTON_BORON_FUSION_CROSS_SECTION_H +#endif // WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 0b51d6b4b61..375cc1e6d51 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H -#define PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#ifndef WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#define WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H #include "TwoProductFusionUtil.H" #include "Particles/WarpXParticleContainer.H" @@ -177,4 +177,4 @@ namespace { } -#endif // PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#endif // WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 07e0174438b..9bd4441eaa2 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SINGLE_NUCLEAR_FUSION_EVENT_H_ -#define SINGLE_NUCLEAR_FUSION_EVENT_H_ +#ifndef WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ +#define WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ #include "BoschHaleFusionCrossSection.H" #include "ProtonBoronFusionCrossSection.H" @@ -131,4 +131,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } -#endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ +#endif // WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index 52e9db8aa94..6ca87b9390d 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H -#define TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#ifndef WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#define WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H #include "TwoProductFusionUtil.H" #include "Particles/WarpXParticleContainer.H" @@ -96,4 +96,4 @@ namespace { } } -#endif // TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#endif // WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H index 09afe1e13a2..dbdef531a16 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TWO_PRODUCT_FUSION_UTIL_H -#define TWO_PRODUCT_FUSION_UTIL_H +#ifndef WARPX_TWO_PRODUCT_FUSION_UTIL_H +#define WARPX_TWO_PRODUCT_FUSION_UTIL_H #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -165,4 +165,4 @@ namespace { } } -#endif // TWO_PRODUCT_FUSION_UTIL_H +#endif // WARPX_TWO_PRODUCT_FUSION_UTIL_H diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index d53ce4348fb..3cb7197b93a 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLE_CREATION_FUNC_H_ -#define PARTICLE_CREATION_FUNC_H_ +#ifndef WARPX_PARTICLE_CREATION_FUNC_H_ +#define WARPX_PARTICLE_CREATION_FUNC_H_ #include "BinaryCollisionUtils.H" @@ -15,7 +15,6 @@ #include "Particles/ParticleCreation/SmartCopy.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" -#include "WarpX.H" #include #include @@ -52,7 +51,7 @@ public: * @param[in] collision_name the name of the collision * @param[in] mypc pointer to the MultiParticleContainer */ - ParticleCreationFunc (std::string collision_name, MultiParticleContainer const * mypc); + ParticleCreationFunc (const std::string& collision_name, MultiParticleContainer const * mypc); /** * \brief operator() of the ParticleCreationFunc class. It creates new particles from binary @@ -68,8 +67,7 @@ public: * function specific to the considered binary collision. * * @param[in] n_total_pairs how many binary collisions have been performed in this tile - * @param[in, out] soa_1 struct of array data of the first colliding particle species - * @param[in, out] soa_2 struct of array data of the second colliding particle species + * @param[in, out] ptile1,ptile2 the particle tiles of the two colliding species * @param[out] tile_products array containing tile data of the product particles. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species @@ -95,7 +93,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& n_total_pairs, - const SoaData_type& soa_1, const SoaData_type& soa_2, + ParticleTileType& ptile1, ParticleTileType& ptile2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, @@ -130,6 +128,9 @@ public: tile_products[i]->resize(products_np[i] + num_added); } + const auto soa_1 = ptile1.getParticleTileData(); + const auto soa_2 = ptile2.getParticleTileData(); + amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; @@ -197,37 +198,10 @@ public: } // Remove p_pair_reaction_weight[i] from the colliding particles' weights - amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], - -p_pair_reaction_weight[i]); - amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], - -p_pair_reaction_weight[i]); - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; -#else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); -#endif - }; - - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to -1 - if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) - { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); - - } - if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) - { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); - } + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); // Initialize the product particles' momentum, using a function depending on the // specific collision type @@ -318,13 +292,13 @@ class NoParticleCreationFunc public: NoParticleCreationFunc () = default; - NoParticleCreationFunc (const std::string /*collision_name*/, + NoParticleCreationFunc (const std::string& /*collision_name*/, MultiParticleContainer const * const /*mypc*/) {} AMREX_INLINE amrex::Vector operator() ( const index_type& /*n_total_pairs*/, - const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + ParticleTileType& /*ptile1*/, ParticleTileType& /*ptile2*/, amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, @@ -339,4 +313,4 @@ public: } }; -#endif // PARTICLE_CREATION_FUNC_H_ +#endif // WARPX_PARTICLE_CREATION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 33629cd530f..3690c5d7295 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -17,13 +17,12 @@ #include -ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, - MultiParticleContainer const * const mypc) +ParticleCreationFunc::ParticleCreationFunc (const std::string& collision_name, + MultiParticleContainer const * const mypc): + m_collision_type{BinaryCollisionUtils::get_collision_type(collision_name, mypc)} { const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) { // Proton-Boron fusion only produces alpha particles diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index db79eaf1a01..21a5de6a38c 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -18,7 +18,7 @@ class CollisionBase { public: - CollisionBase (std::string collision_name); + CollisionBase (const std::string& collision_name); virtual void doCollisions (amrex::Real /*cur_time*/, amrex::Real /*dt*/, MultiParticleContainer* /*mypc*/ ){} diff --git a/Source/Particles/Collision/CollisionBase.cpp b/Source/Particles/Collision/CollisionBase.cpp index 8f799c83013..c8b4f76cbd2 100644 --- a/Source/Particles/Collision/CollisionBase.cpp +++ b/Source/Particles/Collision/CollisionBase.cpp @@ -10,7 +10,7 @@ #include -CollisionBase::CollisionBase (std::string collision_name) +CollisionBase::CollisionBase (const std::string& collision_name) { // read collision species diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 5abee34c86a..d723a1017d8 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -8,9 +8,10 @@ #include "Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H" #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" -#include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" -#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" +#include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -45,7 +46,8 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) if (type == "pairwisecoulomb") { allcollisions[i] = std::make_unique>( - collision_names[i], mypc); + collision_names[i], mypc + ); } else if (type == "background_mcc") { allcollisions[i] = std::make_unique(collision_names[i]); @@ -54,12 +56,16 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) allcollisions[i] = std::make_unique(collision_names[i]); } else if (type == "dsmc") { - allcollisions[i] = std::make_unique(collision_names[i]); + allcollisions[i] = + std::make_unique>( + collision_names[i], mypc + ); } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( - collision_names[i], mypc); + collision_names[i], mypc + ); } else{ WARPX_ABORT_WITH_MESSAGE("Unknown collision type."); diff --git a/Source/Particles/Collision/ScatteringProcess.H b/Source/Particles/Collision/ScatteringProcess.H index d05ea32e2fe..59ef7a02afb 100644 --- a/Source/Particles/Collision/ScatteringProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -57,7 +57,7 @@ public: */ static void readCrossSectionFile ( - std::string cross_section_file, + const std::string& cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ); diff --git a/Source/Particles/Collision/ScatteringProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp index c15d11eea07..ea1b4b40f54 100644 --- a/Source/Particles/Collision/ScatteringProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -94,7 +94,7 @@ ScatteringProcess::parseProcessType(const std::string& scattering_process) void ScatteringProcess::readCrossSectionFile ( - const std::string cross_section_file, + const std::string& cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) { diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index d0822789015..2912b769c57 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef CHARGEDEPOSITION_H_ -#define CHARGEDEPOSITION_H_ +#ifndef WARPX_CHARGEDEPOSITION_H_ +#define WARPX_CHARGEDEPOSITION_H_ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" @@ -33,8 +33,6 @@ * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost: Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, @@ -42,20 +40,14 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const int* ion_lev, amrex::FArrayBox& rho_fab, long np_to_deposit, - const std::array& dx, + const std::array& dx, const std::array xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -87,21 +79,9 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into rho_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0.; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif // --- Get particle quantities amrex::Real wq = q*wp[ip]*invvol; if (do_ionization){ @@ -202,13 +182,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #ifndef WARPX_DIM_RZ amrex::ignore_unused(n_rz_azimuthal_modes); @@ -230,8 +203,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. * \param a_bins * \param box * \param geom @@ -245,13 +216,11 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio amrex::FArrayBox& rho_fab, const amrex::IntVect& ix_type, const long np_to_deposit, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, const int n_rz_azimuthal_modes, - amrex::Real* cost, - const long load_balance_costs_update_algo, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -264,7 +233,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) - amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); + amrex::ignore_unused(ix_type, a_bins, box, geom, a_tbox_max_size, bin_size); #endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer @@ -299,14 +268,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into rho_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0.; - } -#endif - #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); @@ -394,11 +355,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio { const unsigned int ip = permutation[ip_orig]; -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif // --- Get particle quantities amrex::Real wq = q*wp[ip]*invvol; if (do_ionization){ @@ -506,17 +462,10 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif // defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #ifndef WARPX_DIM_RZ amrex::ignore_unused(n_rz_azimuthal_modes); #endif } -#endif // CHARGEDEPOSITION_H_ +#endif // WARPX_CHARGEDEPOSITION_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 8d4c5b6a5f4..3dce7d28534 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef CURRENTDEPOSITION_H_ -#define CURRENTDEPOSITION_H_ +#ifndef WARPX_CURRENTDEPOSITION_H_ +#define WARPX_CURRENTDEPOSITION_H_ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" @@ -287,8 +287,6 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionShapeN (const GetParticlePosition& GetPosition, @@ -306,9 +304,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, const std::array& xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex::literals; @@ -316,10 +312,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -354,22 +346,9 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jz_type = jz_fab.box().type(); // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - amrex::ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); @@ -395,13 +374,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -424,8 +396,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, @@ -445,19 +415,13 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, const std::array& xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real* cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex::literals; #if !defined(WARPX_DIM_RZ) amrex::ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -490,22 +454,9 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, amrex::IntVect const jz_type = jz_fab.box().type(); // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - amrex::ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); @@ -539,13 +490,6 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -570,8 +514,6 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, @@ -590,8 +532,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::Dim3 lo, amrex::Real q, int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -616,13 +556,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); const auto domain = geom.Domain(); @@ -651,11 +584,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::launch( nblocks, threads_per_block, shared_mem_bytes, amrex::Gpu::gpuStream(), [=] AMREX_GPU_DEVICE () noexcept { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif const int bin_id = blockIdx.x; const unsigned int bin_start = offsets_ptr[bin_id]; const unsigned int bin_stop = offsets_ptr[bin_id+1]; @@ -743,18 +671,11 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, __syncthreads(); addLocalToGlobal(tbox_z, jz_arr, jz_buff); }); -#if defined(WARPX_USE_GPUCLOCK) - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #else // not using hip/cuda // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -782,8 +703,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -802,9 +721,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, std::array xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real * const cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex; using namespace amrex::literals; @@ -813,10 +730,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -857,22 +770,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - // --- Get particle quantities Real const gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq @@ -1127,13 +1027,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -1159,8 +1052,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, @@ -1180,23 +1071,17 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real * const cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex; #if !defined(WARPX_DIM_RZ) ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -1236,22 +1121,9 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - #if !defined(WARPX_DIM_3D) constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); @@ -1526,13 +1398,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -1560,8 +1425,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, @@ -1581,23 +1444,17 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real * const cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex; #if !defined(WARPX_DIM_RZ) ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -1628,21 +1485,9 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif #if !defined(WARPX_DIM_3D) constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); @@ -2190,13 +2035,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -2224,9 +2062,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ * \param[in] lo Dimension-agnostic lower bounds of index domain * \param[in] q Species charge * \param[in] n_rz_azimuthal_modes Number of azimuthal modes in RZ geometry - * \param[in,out] cost Pointer to (load balancing) cost corresponding to box where - present particles deposit current - * \param[in] load_balance_costs_update_algo Selected method for updating load balance costs */ template void doVayDepositionShapeN (const GetParticlePosition& GetPosition, @@ -2245,9 +2080,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, const std::array& xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex::literals; @@ -2265,10 +2098,6 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); #endif -#if !defined(WARPX_USE_GPUCLOCK) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) amrex::ignore_unused(n_rz_azimuthal_modes); @@ -2319,21 +2148,8 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Array4 const& Dz_arr = Dz_fab.array(); // Loop over particles and deposit (Dx,Dy,Dz) into Dx_fab, Dy_fab and Dz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor(np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - // Inverse of Lorentz factor gamma const amrex::Real invgam = 1._rt / std::sqrt(1._rt + uxp[ip] * uxp[ip] * invcsq + uyp[ip] * uyp[ip] * invcsq @@ -2560,14 +2376,6 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, // Synchronize so that temp_fab can be safely deallocated in its destructor amrex::Gpu::streamSynchronize(); - -# if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -# endif #endif // #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) } -#endif // CURRENTDEPOSITION_H_ +#endif // WARPX_CURRENTDEPOSITION_H_ diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index e28835a57df..1e2294be3a2 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SHAREDDEPOSITIONUTILS_H_ -#define SHAREDDEPOSITIONUTILS_H_ +#ifndef WARPX_SHAREDDEPOSITIONUTILS_H_ +#define WARPX_SHAREDDEPOSITIONUTILS_H_ #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/ShapeFactors.H" @@ -281,4 +281,4 @@ void depositComponent (const GetParticlePosition& GetPosition, } #endif -#endif // SHAREDDEPOSITIONUTILS_H_ +#endif // WARPX_SHAREDDEPOSITIONUTILS_H_ diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 61aeb19aea1..cee7ec07eb5 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef IONIZATION_H_ -#define IONIZATION_H_ +#ifndef WARPX_IONIZATION_H_ +#define WARPX_IONIZATION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -171,4 +171,4 @@ struct IonizationTransformFunc } }; -#endif +#endif //WARPX_IONIZATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index fb723f0b79a..d550c5f03ee 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_PAIR_GENERATION_H_ -#define QED_PAIR_GENERATION_H_ +#ifndef WARPX_QED_PAIR_GENERATION_H_ +#define WARPX_QED_PAIR_GENERATION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -208,4 +208,4 @@ private: amrex::Dim3 m_lo; }; -#endif //QED_PAIR_GENERATION_H_ +#endif //WARPX_QED_PAIR_GENERATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 567b260d0e4..dd5f57a8f24 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_PHOTON_EMISSION_H_ -#define QED_PHOTON_EMISSION_H_ +#ifndef WARPX_QED_PHOTON_EMISSION_H_ +#define WARPX_QED_PHOTON_EMISSION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -267,4 +267,4 @@ void cleanLowEnergyPhotons( } -#endif //QED_PHOTON_EMISSION_H_ +#endif //WARPX_QED_PHOTON_EMISSION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H index bf4c566b3ba..32b58dc50dc 100644 --- a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H +++ b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_SCHWINGER_PROCESS_H_ -#define QED_SCHWINGER_PROCESS_H_ +#ifndef WARPX_QED_SCHWINGER_PROCESS_H_ +#define WARPX_QED_SCHWINGER_PROCESS_H_ #include "Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H" #include "Utils/TextMsg.H" @@ -95,4 +95,4 @@ struct SchwingerTransformFunc } }; -#endif // QED_SCHWINGER_PROCESS_H_ +#endif // WARPX_QED_SCHWINGER_PROCESS_H_ diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index e88774f642c..fc68e8d9723 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FILTERFUNCTORS_H -#define FILTERFUNCTORS_H +#ifndef WARPX_FILTERFUNCTORS_H +#define WARPX_FILTERFUNCTORS_H #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/WarpXParticleContainer.H" @@ -216,4 +216,4 @@ private: const amrex::RealBox m_domain; }; -#endif // FILTERFUNCTORS_H +#endif // WARPX_FILTERFUNCTORS_H diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index e51cfa41c58..b5bd4376ba1 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FIELDGATHER_H_ -#define FIELDGATHER_H_ +#ifndef WARPX_FIELDGATHER_H_ +#define WARPX_FIELDGATHER_H_ #include "Particles/Gather/GetExternalFields.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -1852,4 +1852,4 @@ void doGatherShapeNImplicit ( } } -#endif // FIELDGATHER_H_ +#endif // WARPX_FIELDGATHER_H_ diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 4cddfa5a342..5731bc047f4 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -60,4 +60,4 @@ struct ScaleFields } }; -#endif +#endif //WARPX_PARTICLES_GATHER_SCALEFIELDS_H_ diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 9946a680fcd..f007d5088e3 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -213,6 +213,8 @@ public: void defineAllParticleTiles (); + void deleteInvalidParticles (); + void RedistributeLocal (int num_ghost); /** Apply BC. For now, just discard particles outside the domain, regardless @@ -249,7 +251,7 @@ public: * \param[in] species_name The species for which back-transformed particles is set. * \param[in] do_back_transformed_particles The parameter to set if back-transformed particles are set to true/false */ - void SetDoBackTransformedParticles (std::string species_name, bool do_back_transformed_particles); + void SetDoBackTransformedParticles (const std::string& species_name, bool do_back_transformed_particles); [[nodiscard]] int nSpeciesDepositOnMainGrid () const { @@ -294,7 +296,7 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - void ScrapeParticles (const amrex::Vector& distance_to_eb); + void ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb); std::string m_B_ext_particle_s = "none"; std::string m_E_ext_particle_s = "none"; @@ -330,7 +332,7 @@ public: const amrex::MultiFab& Bz); #endif - [[nodiscard]] int getSpeciesID (std::string product_str) const; + [[nodiscard]] int getSpeciesID (const std::string& product_str) const; amrex::Vector>::iterator begin() {return allcontainers.begin();} amrex::Vector>::iterator end() {return allcontainers.end();} diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a31f426a0e4..fc496217388 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -10,6 +10,8 @@ * License: BSD-3-Clause-LBNL */ #include "MultiParticleContainer.H" + +#include "FieldSolver/Fields.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" @@ -80,6 +82,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { @@ -632,6 +635,14 @@ MultiParticleContainer::defineAllParticleTiles () } } +void +MultiParticleContainer::deleteInvalidParticles () +{ + for (auto& pc : allcontainers) { + pc->deleteInvalidParticles(); + } +} + void MultiParticleContainer::RedistributeLocal (const int num_ghost) { @@ -803,7 +814,7 @@ MultiParticleContainer::mapSpeciesProduct () /* \brief Given a species name, return its ID. */ int -MultiParticleContainer::getSpeciesID (std::string product_str) const +MultiParticleContainer::getSpeciesID (const std::string& product_str) const { auto species_and_lasers_names = GetSpeciesAndLasersNames(); int i_product = 0; @@ -832,7 +843,7 @@ MultiParticleContainer::SetDoBackTransformedParticles (const bool do_back_transf } void -MultiParticleContainer::SetDoBackTransformedParticles (std::string species_name, const bool do_back_transformed_particles) { +MultiParticleContainer::SetDoBackTransformedParticles (const std::string& species_name, const bool do_back_transformed_particles) { auto species_names_list = GetSpeciesNames(); bool found = false; // Loop over species @@ -945,11 +956,11 @@ void MultiParticleContainer::CheckIonizationProductSpecies() } } -void MultiParticleContainer::ScrapeParticles (const amrex::Vector& distance_to_eb) +void MultiParticleContainer::ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb) { #ifdef AMREX_USE_EB for (auto& pc : allcontainers) { - scrapeParticles(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); } #else amrex::ignore_unused(distance_to_eb); @@ -1344,12 +1355,12 @@ MultiParticleContainer::doQEDSchwinger () pc_product_ele->defineAllParticleTiles(); pc_product_pos->defineAllParticleTiles(); - const MultiFab & Ex = warpx.getEfield(level_0,0); - const MultiFab & Ey = warpx.getEfield(level_0,1); - const MultiFab & Ez = warpx.getEfield(level_0,2); - const MultiFab & Bx = warpx.getBfield(level_0,0); - const MultiFab & By = warpx.getBfield(level_0,1); - const MultiFab & Bz = warpx.getBfield(level_0,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, level_0,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, level_0,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, level_0,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, level_0,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, level_0,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, level_0,2); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -1388,10 +1399,12 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; + const amrex::Geometry& geom_level_zero = warpx.Geom(level_0); + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, - Transform); + Transform, geom_level_zero); setNewParticleIDs(dst_ele_tile, np_ele_dst, num_added); setNewParticleIDs(dst_pos_tile, np_pos_dst, num_added); diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index e7a7a20fad5..02f4c44314a 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef NamedComponentParticleContainer_H_ -#define NamedComponentParticleContainer_H_ +#ifndef WARPX_NamedComponentParticleContainer_H_ +#define WARPX_NamedComponentParticleContainer_H_ #include "Utils/TextMsg.H" @@ -190,6 +190,19 @@ public: } } + void defineAllParticleTiles () noexcept + { + for (int lev = 0; lev <= amrex::ParticleContainerPureSoA::finestLevel(); ++lev) + { + for (auto mfi = amrex::ParticleContainerPureSoA::MakeMFIter(lev); mfi.isValid(); ++mfi) + { + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + amrex::ParticleContainerPureSoA::DefineAndReturnParticleTile(lev, grid_id, tile_id); + } + } + } + /** Return the name-to-index map for the compile-time and runtime-time real components */ [[nodiscard]] std::map getParticleComps () const noexcept { return particle_comps;} /** Return the name-to-index map for the compile-time and runtime-time integer components */ @@ -206,4 +219,4 @@ protected: std::map particle_runtime_icomps; }; -#endif +#endif //WARPX_NamedComponentParticleContainer_H_ diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index 78b090056b1..a1b31f61cd2 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARIES_H_ -#define PARTICLEBOUNDARIES_H_ +#ifndef WARPX_PARTICLEBOUNDARIES_H_ +#define WARPX_PARTICLEBOUNDARIES_H_ #include "Utils/WarpXAlgorithmSelection.H" @@ -22,6 +22,8 @@ struct ParticleBoundaries void Set_reflect_all_velocities (bool flag); void SetAll (ParticleBoundaryType bc); + /** Sets thermal velocity in ParticleBoundariesData 'data.m_uth' to u_th */ + void SetThermalVelocity (amrex::Real u_th); void SetBoundsX (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); void SetBoundsY (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); @@ -54,6 +56,7 @@ struct ParticleBoundaries ParticleBoundaryType ymax_bc; ParticleBoundaryType zmin_bc; ParticleBoundaryType zmax_bc; + amrex::Real m_uth = 0.; amrex::ParserExecutor<1> reflection_model_xlo; amrex::ParserExecutor<1> reflection_model_xhi; @@ -67,4 +70,4 @@ struct ParticleBoundaries ParticleBoundariesData data; }; -#endif /*PARTICLEBOUNDARIES_H_*/ +#endif /*WARPX_PARTICLEBOUNDARIES_H_*/ diff --git a/Source/Particles/ParticleBoundaries.cpp b/Source/Particles/ParticleBoundaries.cpp index 5b51fa3fd25..10268a9c01f 100644 --- a/Source/Particles/ParticleBoundaries.cpp +++ b/Source/Particles/ParticleBoundaries.cpp @@ -6,6 +6,7 @@ */ #include "ParticleBoundaries.H" +#include "WarpX.H" #include "Utils/Parser/ParserUtils.H" @@ -32,6 +33,12 @@ ParticleBoundaries::SetAll (ParticleBoundaryType bc) data.zmax_bc = bc; } +void +ParticleBoundaries::SetThermalVelocity (amrex::Real u_th) +{ + data.m_uth = u_th; +} + void ParticleBoundaries::SetBoundsX (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi) { diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index bbec1f54e01..71b95aae95a 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -1,25 +1,26 @@ -/* Copyright 2021 David Grote +/* Copyright 2021 David Grote, Remi Lehe, Revathi Jambunathan * * This file is part of WarpX. * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARIES_K_H_ -#define PARTICLEBOUNDARIES_K_H_ +#ifndef WARPX_PARTICLEBOUNDARIES_K_H_ +#define WARPX_PARTICLEBOUNDARIES_K_H_ #include "ParticleBoundaries.H" +#include "Initialization/SampleGaussianFluxDistribution.H" #include namespace ApplyParticleBoundaries { - + using namespace amrex::literals; /* \brief Applies the boundary condition on a specific axis * This is called by apply_boundaries. */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void apply_boundary (amrex::ParticleReal& x, amrex::Real xmin, amrex::Real xmax, - bool& change_sign_ux, bool& particle_lost, + bool& change_sign_ux, bool& rethermalize_x, bool& particle_lost, ParticleBoundaryType xmin_bc, ParticleBoundaryType xmax_bc, amrex::Real refl_probability_xmin, amrex::Real refl_probability_xmax, amrex::RandomEngine const& engine ) @@ -42,6 +43,10 @@ namespace ApplyParticleBoundaries { x = 2*xmin - x; change_sign_ux = true; } + else if (xmin_bc == ParticleBoundaryType::Thermal) { + x = 2*xmin - x; + rethermalize_x = true; + } } else if (x > xmax) { if (xmax_bc == ParticleBoundaryType::Open) { @@ -61,16 +66,38 @@ namespace ApplyParticleBoundaries { x = 2*xmax - x; change_sign_ux = true; } + else if (xmax_bc == ParticleBoundaryType::Thermal) { + x = 2*xmax - x; + rethermalize_x = true; + } } } - /* \brief Applies absorbing or reflecting boundary condition to the input particles, along all axis. + /* \brief Thermalize particles that have been identified to cross the boundary. + * The normal component samples from a half-Maxwellian, + * and the two tangential components will sample from full Maxwellian disbutions + * with thermal velocity uth + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void thermalize_boundary_particle (amrex::ParticleReal& u_norm, amrex::ParticleReal& u_tang1, + amrex::ParticleReal& u_tang2, amrex::Real uth, + amrex::RandomEngine const& engine) + { + u_tang1 = (uth > 0._rt) ? PhysConst::c * amrex::RandomNormal(0._rt, uth, engine) : 0._rt; + u_tang2 = (uth > 0._rt) ? PhysConst::c * amrex::RandomNormal(0._rt, uth, engine) : 0._rt; + u_norm = (uth > 0._rt) ? std::copysign(1._prt, -u_norm) * PhysConst::c * generateGaussianFluxDist(0._rt, uth, engine) : 0._rt; + } + + + /* \brief Applies absorbing, reflecting or thermal boundary condition to the input particles, along all axis. * For reflecting boundaries, the position of the particle is changed appropriately and * the sign of the velocity is changed (depending on the reflect_all_velocities flag). * For absorbing, a flag is set whether the particle has been lost (it is up to the calling * code to take appropriate action to remove any lost particles). Absorbing boundaries can * be given a reflection coefficient for stochastic reflection of particles, this * coefficient is zero by default. + * For thermal boundaries, the particle is first reflected and the position of the particle + * is changed appropriately. * Note that periodic boundaries are handled in AMReX code. * * \param x, xmin, xmax: particle x position, location of x boundary @@ -103,21 +130,33 @@ namespace ApplyParticleBoundaries { bool change_sign_uz = false; #ifndef WARPX_DIM_1D_Z - apply_boundary(x, xmin, xmax, change_sign_ux, particle_lost, + bool rethermalize_x = false; // stores if particle crosses x boundary and needs to be thermalized + apply_boundary(x, xmin, xmax, change_sign_ux, rethermalize_x, particle_lost, boundaries.xmin_bc, boundaries.xmax_bc, boundaries.reflection_model_xlo(-ux), boundaries.reflection_model_xhi(ux), engine); + if (rethermalize_x) { + thermalize_boundary_particle(ux, uy, uz, boundaries.m_uth, engine); + } #endif #ifdef WARPX_DIM_3D - apply_boundary(y, ymin, ymax, change_sign_uy, particle_lost, + bool rethermalize_y = false; // stores if particle crosses y boundary and needs to be thermalized + apply_boundary(y, ymin, ymax, change_sign_uy, rethermalize_y, particle_lost, boundaries.ymin_bc, boundaries.ymax_bc, boundaries.reflection_model_ylo(-uy), boundaries.reflection_model_yhi(uy), engine); + if (rethermalize_y) { + thermalize_boundary_particle(uy, uz, ux, boundaries.m_uth, engine); + } #endif - apply_boundary(z, zmin, zmax, change_sign_uz, particle_lost, + bool rethermalize_z = false; // stores if particle crosses z boundary and needs to be thermalized + apply_boundary(z, zmin, zmax, change_sign_uz, rethermalize_z, particle_lost, boundaries.zmin_bc, boundaries.zmax_bc, boundaries.reflection_model_zlo(-uz), boundaries.reflection_model_zhi(uz), engine); + if (rethermalize_z) { + thermalize_boundary_particle(uz, ux, uy, boundaries.m_uth, engine); + } if (boundaries.reflect_all_velocities && (change_sign_ux | change_sign_uy | change_sign_uz)) { change_sign_ux = true; @@ -147,4 +186,4 @@ namespace ApplyParticleBoundaries { } } -#endif +#endif //WARPX_PARTICLEBOUNDARIES_K_H_ diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 1e9748b2ff5..d33834309ab 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARYBUFFER_H_ -#define PARTICLEBOUNDARYBUFFER_H_ +#ifndef WARPX_PARTICLEBOUNDARYBUFFER_H_ +#define WARPX_PARTICLEBOUNDARYBUFFER_H_ #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer.H" @@ -39,8 +39,10 @@ public: const std::vector& getSpeciesNames() const; - void gatherParticles (MultiParticleContainer& mypc, - const amrex::Vector& distance_to_eb); + void gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc); + void gatherParticlesFromEmbeddedBoundaries ( + MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb + ); void redistribute (); void clearParticles (); @@ -48,11 +50,11 @@ public: void printNumParticles () const; - int getNumParticlesInContainer(std::string species_name, int boundary, bool local); + int getNumParticlesInContainer(const std::string& species_name, int boundary, bool local); - PinnedMemoryParticleContainer& getParticleBuffer(std::string species_name, int boundary); + PinnedMemoryParticleContainer& getParticleBuffer(const std::string& species_name, int boundary); - PinnedMemoryParticleContainer* getParticleBufferPointer(std::string species_name, int boundary); + PinnedMemoryParticleContainer* getParticleBufferPointer(const std::string& species_name, int boundary); static constexpr int numBoundaries () { return AMREX_SPACEDIM*2 @@ -80,4 +82,4 @@ private: mutable std::vector m_species_names; }; -#endif /*PARTICLEBOUNDARYBUFFER_H_*/ +#endif /*WARPX_PARTICLEBOUNDARYBUFFER_H_*/ diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index d1f9f814bc5..345a0bfa592 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -98,7 +98,8 @@ struct FindEmbeddedBoundaryIntersection { amrex::Real W[AMREX_SPACEDIM][2]; amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); return phi_value; } ); @@ -115,11 +116,12 @@ struct FindEmbeddedBoundaryIntersection { // record the components of the normal on the destination int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); int ic, jc, kc; // Cell-centered indices - int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weight - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc, nodal=0); // nodal=0 to calculate the weights with respect to the cell-centered nodes + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phiarr, dxi); DistanceToEB::normalize(normal); @@ -354,8 +356,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { } } -void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, - const amrex::Vector& distance_to_eb) +void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc) { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); @@ -439,10 +440,19 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } } } +} +void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( + MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb) +{ #ifdef AMREX_USE_EB WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); + using PIter = amrex::ParConstIterSoA; + const auto& warpx_instance = WarpX::GetInstance(); + const amrex::Geometry& geom = warpx_instance.Geom(0); + auto plo = geom.ProbLoArray(); + auto& buffer = m_particle_containers[m_particle_containers.size()-1]; for (int i = 0; i < numSpecies(); ++i) { @@ -526,12 +536,12 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } } #else - amrex::ignore_unused(distance_to_eb); + amrex::ignore_unused(mypc, distance_to_eb); #endif } int ParticleBoundaryBuffer::getNumParticlesInContainer( - const std::string species_name, int boundary, bool local) { + const std::string& species_name, int boundary, bool local) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); @@ -545,7 +555,7 @@ int ParticleBoundaryBuffer::getNumParticlesInContainer( } PinnedMemoryParticleContainer & -ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int boundary) { +ParticleBoundaryBuffer::getParticleBuffer(const std::string& species_name, int boundary) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); @@ -560,7 +570,7 @@ ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int bo } PinnedMemoryParticleContainer * -ParticleBoundaryBuffer::getParticleBufferPointer(const std::string species_name, int boundary) { +ParticleBoundaryBuffer::getParticleBufferPointer(const std::string& species_name, int boundary) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 3f0cbab79d5..88b23905481 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef DEFAULTINITIALIZATION_H_ -#define DEFAULTINITIALIZATION_H_ +#ifndef WARPX_DEFAULTINITIALIZATION_H_ +#define WARPX_DEFAULTINITIALIZATION_H_ #include #ifdef WARPX_QED @@ -280,4 +280,4 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, } -#endif +#endif //WARPX_DEFAULTINITIALIZATION_H_ diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 095ebbf6a85..4815a98ca31 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FILTER_COPY_TRANSFORM_H_ -#define FILTER_COPY_TRANSFORM_H_ +#ifndef WARPX_FILTER_COPY_TRANSFORM_H_ +#define WARPX_FILTER_COPY_TRANSFORM_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" @@ -360,4 +360,4 @@ Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi std::forward(transform)); } -#endif +#endif //WARPX_FILTER_COPY_TRANSFORM_H_ diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 2a4c9fccad0..8a83c60b221 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -5,11 +5,10 @@ * License: BSD-3-Clause-LBNL */ -#ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ -#define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#ifndef WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#define WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" -#include "WarpX.H" #include #include @@ -40,6 +39,7 @@ * \param[in] create1 callable that defines what will be done for the create step for dst1. * \param[in] create2 callable that defines what will be done for the create step for dst2. * \param[in] transform callable that defines the transformation to apply on dst1 and dst2. + * \param[in] geom_lev_zero the geometry object associated to level zero * * \return num_added the number of particles that were written to dst1 and dst2. */ @@ -51,34 +51,30 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, const FAB *src_FAB, const Index* mask, const Index dst1_index, const Index dst2_index, CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept + TransFunc&& transform, const amrex::Geometry& geom_lev_zero) noexcept { using namespace amrex; const auto ncells = box.volume(); if (ncells == 0) { return 0; } - auto & warpx = WarpX::GetInstance(); - const int level_0 = 0; - Geometry const & geom = warpx.Geom(level_0); - constexpr int spacedim = AMREX_SPACEDIM; #if defined(WARPX_DIM_1D_Z) - const Real zlo_global = geom.ProbLo(0); - const Real dz = geom.CellSize(0); + const Real zlo_global = geom_lev_zero.ProbLo(0); + const Real dz = geom_lev_zero.CellSize(0); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const Real xlo_global = geom.ProbLo(0); - const Real dx = geom.CellSize(0); - const Real zlo_global = geom.ProbLo(1); - const Real dz = geom.CellSize(1); + const Real xlo_global = geom_lev_zero.ProbLo(0); + const Real dx = geom_lev_zero.CellSize(0); + const Real zlo_global = geom_lev_zero.ProbLo(1); + const Real dz = geom_lev_zero.CellSize(1); #elif defined(WARPX_DIM_3D) - const Real xlo_global = geom.ProbLo(0); - const Real dx = geom.CellSize(0); - const Real ylo_global = geom.ProbLo(1); - const Real dy = geom.CellSize(1); - const Real zlo_global = geom.ProbLo(2); - const Real dz = geom.CellSize(2); + const Real xlo_global = geom_lev_zero.ProbLo(0); + const Real dx = geom_lev_zero.CellSize(0); + const Real ylo_global = geom_lev_zero.ProbLo(1); + const Real dy = geom_lev_zero.CellSize(1); + const Real zlo_global = geom_lev_zero.ProbLo(2); + const Real dz = geom_lev_zero.CellSize(2); #endif const auto arrNumPartCreation = src_FAB->array(); @@ -199,6 +195,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, * \param[in] create1 callable that defines what will be done for the create step for dst1. * \param[in] create2 callable that defines what will be done for the create step for dst2. * \param[in] transform callable that defines the transformation to apply on dst1 and dst2. + * \param[in] geom_lev_zero the geometry object associated to level zero * * \return num_added the number of particles that were written to dst1 and dst2. */ @@ -209,7 +206,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc && transform) noexcept + TransFunc && transform, const amrex::Geometry& geom_lev_zero) noexcept { using namespace amrex; @@ -241,7 +238,8 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), - std::forward(transform)); + std::forward(transform), + geom_lev_zero); } -#endif // FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#endif // WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 6a6ceb3d290..e1d944e9c30 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SMART_COPY_H_ -#define SMART_COPY_H_ +#ifndef WARPX_SMART_COPY_H_ +#define WARPX_SMART_COPY_H_ #include "DefaultInitialization.H" #include "SmartUtils.H" @@ -163,4 +163,4 @@ public: [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; -#endif +#endif //WARPX_SMART_COPY_H_ diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index b4f25d5daad..fe4cb5929e0 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SMART_CREATE_H_ -#define SMART_CREATE_H_ +#ifndef WARPX_SMART_CREATE_H_ +#define WARPX_SMART_CREATE_H_ #include "DefaultInitialization.H" @@ -112,4 +112,4 @@ public: [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; -#endif //SMART_CREATE_H_ +#endif //WARPX_SMART_CREATE_H_ diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index dbac563ca28..652a3aecd17 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SMART_UTILS_H_ -#define SMART_UTILS_H_ +#ifndef WARPX_SMART_UTILS_H_ +#define WARPX_SMART_UTILS_H_ #include "DefaultInitialization.H" @@ -70,4 +70,4 @@ void setNewParticleIDs (PTile& ptile, amrex::Long old_size, amrex::Long num_adde }); } -#endif //SMART_UTILS_H_ +#endif //WARPX_SMART_UTILS_H_ diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index c417f7467b3..d5fc68f4097 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -4,10 +4,14 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEIO_H_ -#define PARTICLEIO_H_ +#ifndef WARPX_PARTICLEIO_H_ +#define WARPX_PARTICLEIO_H_ #include "Particles/WarpXParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/PinnedMemoryParticleContainer.H" + +#include #include #include @@ -77,4 +81,15 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer* } } -#endif /* PARTICLEIO_H_ */ +/** Gathers phi (electrostatic potential) from a MultiFab to the macroparticles. + * Adds a runtime component of the particle container to store it. + * + * @param tmp the particle container on which to store the gathered field + * @param electrostatic_solver_id the type of electrostatic solver used + * @param is_full_diagnostic whether this diagnostic is a full diagnostic + */ +void +storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, + int electrostatic_solver_id, bool is_full_diagnostic ); + +#endif /* WARPX_PARTICLEIO_H_ */ diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index b77c1147a15..5d9b41b8b75 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -226,13 +226,7 @@ public: amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, amrex::Real t_lab = 0.) const; - void AddGaussianBeam ( - PlasmaInjector const& plasma_injector, - amrex::Real x_m, amrex::Real y_m, amrex::Real z_m, - amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, - amrex::Real x_cut, amrex::Real y_cut, amrex::Real z_cut, - amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order, - amrex::Real focal_distance); + void AddGaussianBeam (PlasmaInjector const& plasma_injector); /** Load a particle beam from an external file * @param[in] the PlasmaInjector instance holding the input parameters @@ -358,14 +352,14 @@ public: * @param[in] ptr the pointer */ void set_breit_wheeler_engine_ptr - (std::shared_ptr ptr) override; + (const std::shared_ptr& ptr) override; /** * Acquires a shared smart pointer to a QuantumSynchrotronEngine * @param[in] ptr the pointer */ void set_quantum_sync_engine_ptr - (std::shared_ptr ptr) override; + (const std::shared_ptr& ptr) override; //__________ BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b168194da62..e48645c70a1 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -148,7 +148,9 @@ namespace AMREX_GPU_HOST_DEVICE PDim3(const amrex::XDim3& a): - x{a.x}, y{a.y}, z{a.z} + x{static_cast(a.x)}, + y{static_cast(a.y)}, + z{static_cast(a.z)} {} AMREX_GPU_HOST_DEVICE @@ -444,6 +446,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_boundary.query("reflect_all_velocities", flag); m_boundary_conditions.Set_reflect_all_velocities(flag); + // currently supports only isotropic thermal distribution + // same distribution is applied to all boundaries + const amrex::ParmParse pp_species_boundary("boundary." + species_name); + if (WarpX::isAnyParticleBoundaryThermal()) { + amrex::Real boundary_uth; + utils::parser::getWithParser(pp_species_boundary,"u_th",boundary_uth); + m_boundary_conditions.SetThermalVelocity(boundary_uth); + } } PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) @@ -521,14 +531,22 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame ( } void -PhysicalParticleContainer::AddGaussianBeam ( - PlasmaInjector const& plasma_injector, - const Real x_m, const Real y_m, const Real z_m, - const Real x_rms, const Real y_rms, const Real z_rms, - const Real x_cut, const Real y_cut, const Real z_cut, - const Real q_tot, long npart, - const int do_symmetrize, - const int symmetrization_order, const Real focal_distance) { +PhysicalParticleContainer::AddGaussianBeam (PlasmaInjector const& plasma_injector){ + + const Real x_m = plasma_injector.x_m; + const Real y_m = plasma_injector.y_m; + const Real z_m = plasma_injector.z_m; + const Real x_rms = plasma_injector.x_rms; + const Real y_rms = plasma_injector.y_rms; + const Real z_rms = plasma_injector.z_rms; + const Real x_cut = plasma_injector.x_cut; + const Real y_cut = plasma_injector.y_cut; + const Real z_cut = plasma_injector.z_cut; + const Real q_tot = plasma_injector.q_tot; + long npart = plasma_injector.npart; + const int do_symmetrize = plasma_injector.do_symmetrize; + const int symmetrization_order = plasma_injector.symmetrization_order; + const Real focal_distance = plasma_injector.focal_distance; // Declare temporary vectors on the CPU Gpu::HostVector particle_x; @@ -918,21 +936,7 @@ PhysicalParticleContainer::AddParticles (int lev) } if (plasma_injector->gaussian_beam) { - AddGaussianBeam(*plasma_injector, - plasma_injector->x_m, - plasma_injector->y_m, - plasma_injector->z_m, - plasma_injector->x_rms, - plasma_injector->y_rms, - plasma_injector->z_rms, - plasma_injector->x_cut, - plasma_injector->y_cut, - plasma_injector->z_cut, - plasma_injector->q_tot, - plasma_injector->npart, - plasma_injector->do_symmetrize, - plasma_injector->symmetrization_order, - plasma_injector->focal_distance); + AddGaussianBeam(*plasma_injector); } if (plasma_injector->external_file) { @@ -999,6 +1003,19 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const bool radially_weighted = plasma_injector.radially_weighted; #endif + + // User-defined integer and real attributes: prepare parsers + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); + for (int ia = 0; ia < n_user_int_attribs; ++ia) { + user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); + } + for (int ia = 0; ia < n_user_real_attribs; ++ia) { + user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); + } + MFItInfo info; if (do_tiling && Gpu::notInLaunchRegion()) { info.EnableTiling(tile_size); @@ -1152,19 +1169,13 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); for (int ia = 0; ia < n_user_int_attribs; ++ia) { pa_user_int_pinned[ia] = soa.GetIntData(particle_icomps[m_user_int_attribs[ia]]).data() + old_size; - user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); } for (int ia = 0; ia < n_user_real_attribs; ++ia) { pa_user_real_pinned[ia] = soa.GetRealData(particle_comps[m_user_real_attribs[ia]]).data() + old_size; - user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); } #ifdef AMREX_USE_GPU // To avoid using managed memory, we first define pinned memory vector, initialize on cpu, @@ -1472,7 +1483,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); #endif // The function that calls this is responsible for redistributing particles. @@ -1969,7 +1980,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); #endif // Redistribute the new particles that were added to the temporary container. @@ -2229,6 +2240,8 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit charge after particle push, in component 1 of MultiFab rho. // (Skipped for electrostatic solver, as this may lead to out-of-bounds) if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(rho->nComp() >= 2, + "Cannot deposit charge in rho component 1: only component 0 is allocated!"); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; @@ -2868,11 +2881,11 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, // Using this version of ParallelFor with compile time options // improves performance when qed or external EB are not used by reducing // register pressure. - amrex::ParallelFor(TypeList, - CompileTimeOptions>{}, - {exteb_runtime_flag, qed_runtime_flag}, - np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, - auto qed_control) + amrex::ParallelFor( + TypeList, CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, + [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, auto qed_control) { amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); @@ -3361,11 +3374,7 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose WARPX_PROFILE_VAR_START(blp_resample_actual); if (m_resampler.triggered(timestep, global_numparts)) { - if (verbose) { - amrex::Print() << Utils::TextMsg::Info( - "Resampling " + species_name + " at step " + std::to_string(timestep) - ); - } + Redistribute(); for (int lev = 0; lev <= maxLevel(); lev++) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) @@ -3373,6 +3382,14 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose m_resampler(pti, lev, this); } } + deleteInvalidParticles(); + if (verbose) { + amrex::Print() << Utils::TextMsg::Info( + "Resampled " + species_name + " at step " + std::to_string(timestep) + + ": macroparticle count decreased by " + + std::to_string(static_cast(global_numparts - TotalNumberOfParticles())) + ); + } } WARPX_PROFILE_VAR_STOP(blp_resample_actual); } @@ -3393,14 +3410,14 @@ bool PhysicalParticleContainer::has_breit_wheeler () const void PhysicalParticleContainer:: -set_breit_wheeler_engine_ptr (std::shared_ptr ptr) +set_breit_wheeler_engine_ptr (const std::shared_ptr& ptr) { m_shr_p_bw_engine = ptr; } void PhysicalParticleContainer:: -set_quantum_sync_engine_ptr (std::shared_ptr ptr) +set_quantum_sync_engine_ptr (const std::shared_ptr& ptr) { m_shr_p_qs_engine = ptr; } diff --git a/Source/Particles/PinnedMemoryParticleContainer.H b/Source/Particles/PinnedMemoryParticleContainer.H index eb8fe835c73..402c621eb9a 100644 --- a/Source/Particles/PinnedMemoryParticleContainer.H +++ b/Source/Particles/PinnedMemoryParticleContainer.H @@ -1,8 +1,8 @@ -#ifndef PinnedMemoryParticleContainer_H_ -#define PinnedMemoryParticleContainer_H_ +#ifndef WARPX_PinnedMemoryParticleContainer_H_ +#define WARPX_PinnedMemoryParticleContainer_H_ #include "NamedComponentParticleContainer.H" using PinnedMemoryParticleContainer = NamedComponentParticleContainer; -#endif +#endif //WARPX_PinnedMemoryParticleContainer_H_ diff --git a/Source/Particles/Resampling/CMakeLists.txt b/Source/Particles/Resampling/CMakeLists.txt index 13eb8a8c4b2..8fca0176b86 100644 --- a/Source/Particles/Resampling/CMakeLists.txt +++ b/Source/Particles/Resampling/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) Resampling.cpp ResamplingTrigger.cpp LevelingThinning.cpp + VelocityCoincidenceThinning.cpp ) endforeach() diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index fa05525e270..88e8aa79b4b 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -36,7 +36,13 @@ public: * * @param[in] species_name the name of the resampled species */ - LevelingThinning (std::string species_name); + LevelingThinning (const std::string& species_name); + + /** + * This function queries deprecated input parameters and aborts + * the run if one of them is specified. + */ + void BackwardCompatibility (const std::string& species_name ); /** * \brief A method that performs leveling thinning for the considered species. diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 5dc6a458f97..09531d0e47d 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -29,7 +29,7 @@ #include -LevelingThinning::LevelingThinning (const std::string species_name) +LevelingThinning::LevelingThinning (const std::string& species_name) { using namespace amrex::literals; @@ -47,11 +47,23 @@ LevelingThinning::LevelingThinning (const std::string species_name) } utils::parser::queryWithParser( - pp_species_name, "resampling_algorithm_min_ppc", m_min_ppc); + pp_species_name, "resampling_min_ppc", m_min_ppc); + BackwardCompatibility(species_name); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_min_ppc >= 1, "Resampling min_ppc should be greater than or equal to 1"); } +void LevelingThinning::BackwardCompatibility (const std::string& species_name ) +{ + const amrex::ParmParse pp_species_name(species_name); + int backward_min_ppc; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !pp_species_name.query("resampling_algorithm_min_ppc", backward_min_ppc), + ".resampling_algorithm_min_ppc is no longer a valid option. " + "Please use the renamed option .resampling_min_ppc instead." + ); +} + void LevelingThinning::operator() (WarpXParIter& pti, const int lev, WarpXParticleContainer * const pc) const { diff --git a/Source/Particles/Resampling/Make.package b/Source/Particles/Resampling/Make.package index e70e3446c33..12b58dd9d47 100644 --- a/Source/Particles/Resampling/Make.package +++ b/Source/Particles/Resampling/Make.package @@ -1,5 +1,6 @@ CEXE_sources += Resampling.cpp CEXE_sources += ResamplingTrigger.cpp CEXE_sources += LevelingThinning.cpp +CEXE_sources += VelocityCoincidenceThinning.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Resampling/ diff --git a/Source/Particles/Resampling/Resampling.H b/Source/Particles/Resampling/Resampling.H index 35439e905df..7c5c60d858d 100644 --- a/Source/Particles/Resampling/Resampling.H +++ b/Source/Particles/Resampling/Resampling.H @@ -62,7 +62,7 @@ public: * * @param[in] species_name the name of the resampled species */ - Resampling (std::string species_name); + Resampling (const std::string& species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/Resampling.cpp b/Source/Particles/Resampling/Resampling.cpp index 63ac448880b..15aba88dd3e 100644 --- a/Source/Particles/Resampling/Resampling.cpp +++ b/Source/Particles/Resampling/Resampling.cpp @@ -6,13 +6,14 @@ */ #include "Resampling.H" +#include "VelocityCoincidenceThinning.H" #include "LevelingThinning.H" #include "Utils/TextMsg.H" #include #include -Resampling::Resampling (const std::string species_name) +Resampling::Resampling (const std::string& species_name) { const amrex::ParmParse pp_species_name(species_name); std::string resampling_algorithm_string = "leveling_thinning"; // default resampling algorithm @@ -22,6 +23,10 @@ Resampling::Resampling (const std::string species_name) { m_resampling_algorithm = std::make_unique(species_name); } + else if (resampling_algorithm_string == "velocity_coincidence_thinning") + { + m_resampling_algorithm = std::make_unique(species_name); + } else { WARPX_ABORT_WITH_MESSAGE("Unknown resampling algorithm."); } diff --git a/Source/Particles/Resampling/ResamplingTrigger.H b/Source/Particles/Resampling/ResamplingTrigger.H index 42c3902c247..538680475a0 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.H +++ b/Source/Particles/Resampling/ResamplingTrigger.H @@ -33,7 +33,7 @@ public: * \brief Constructor of the ResamplingTrigger class. Reads the resampling trigger parameters * from the input file. */ - ResamplingTrigger (std::string species_name); + ResamplingTrigger (const std::string& species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/ResamplingTrigger.cpp b/Source/Particles/Resampling/ResamplingTrigger.cpp index 16a32dd7d15..5efb473fe68 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.cpp +++ b/Source/Particles/Resampling/ResamplingTrigger.cpp @@ -14,7 +14,7 @@ #include -ResamplingTrigger::ResamplingTrigger (const std::string species_name) +ResamplingTrigger::ResamplingTrigger (const std::string& species_name) { const amrex::ParmParse pp_species_name(species_name); diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H new file mode 100644 index 00000000000..b8d67d198a4 --- /dev/null +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -0,0 +1,204 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_VELOCITY_COINCIDENCE_THINNING_H_ +#define WARPX_VELOCITY_COINCIDENCE_THINNING_H_ + +#include "Particles/Algorithms/KineticEnergy.H" +#include "Resampling.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" + +/** + * \brief This class implements a particle merging scheme wherein particles + * are clustered in phase space and particles in the same cluster is merged + * into two remaining particles. The scheme conserves linear momentum and + * kinetic energy within each cluster. + */ +class VelocityCoincidenceThinning: public ResamplingAlgorithm { +public: + + /** + * \brief Default constructor of the VelocityCoincidenceThinning class. + */ + VelocityCoincidenceThinning () = default; + + /** + * \brief Constructor of the VelocityCoincidenceThinning class + * + * @param[in] species_name the name of the resampled species + */ + VelocityCoincidenceThinning (const std::string& species_name); + + enum struct VelocityGridType { + Spherical = 0, + Cartesian = 1 + }; + + /** + * \brief A method that performs merging for the considered species. + * + * @param[in] pti WarpX particle iterator of the particles to resample. + * @param[in] lev the index of the refinement level. + * @param[in] pc a pointer to the particle container. + */ + void operator() (WarpXParIter& pti, int lev, WarpXParticleContainer* pc) const final; + + /** + * \brief This merging routine requires functionality to sort a GPU vector + * based on another GPU vector's values. The heap-sort functions below were + * obtained from https://www.geeksforgeeks.org/iterative-heap-sort/ and + * modified for the current purpose. It achieves the same as + * ``` + * std::sort( + * sorted_indices_data + cell_start, sorted_indices_data + cell_stop, + * [&momentum_bin_number_data](size_t i1, size_t i2) { + * return momentum_bin_number_data[i1] < momentum_bin_number_data[i2]; + * } + * ); + * ``` + * but with support for device execution. + */ + struct HeapSort { + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void swap(int &a, int &b) const + { + const auto temp = b; + b = a; + a = temp; + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator() (int index_array[], const int bin_array[], const int start, const int n) const + { + // sort index_array into a max heap structure + for (int i = 1; i < n; i++) + { + auto j = i; + // move child through heap if it is bigger than its parent + while (j > 0 && bin_array[index_array[j+start]] > bin_array[index_array[(j - 1)/2 + start]]) { + // swap child and parent until branch is properly ordered + swap(index_array[j+start], index_array[(j - 1)/2 + start]); + j = (j - 1) / 2; + } + } + + for (int i = n - 1; i > 0; i--) + { + // swap value of first (now the largest value) to the new end point + swap(index_array[start], index_array[i+start]); + + // remake the max heap + int j = 0, index; + while (j < i) { + index = 2 * j + 1; + + // if left child is smaller than right child, point index variable to right child + if (index + 1 < i && bin_array[index_array[index+start]] < bin_array[index_array[index+1+start]]) { + index++; + } + // if parent is smaller than child, swap parent with child having higher value + if (index < i && bin_array[index_array[j+start]] < bin_array[index_array[index+start]]) { + swap(index_array[j+start], index_array[index+start]); + } + j = index; + } + } + } + }; + + /** + * \brief Struct used to assign velocity space bin numbers to a given set + * of particles. + */ + struct VelocityBinCalculator { + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void labelOnSphericalVelocityGrid (const amrex::ParticleReal ux[], + const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], + const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop ) const + { + for (int i = cell_start; i < cell_stop; ++i) + { + // get polar components of the velocity vector + auto u_mag = std::sqrt( + ux[indices[i]]*ux[indices[i]] + + uy[indices[i]]*uy[indices[i]] + + uz[indices[i]]*uz[indices[i]] + ); + auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; + auto u_phi = std::acos(uz[indices[i]]/u_mag); + + const int ii = static_cast(u_theta / dutheta); + const int jj = static_cast(u_phi / duphi); + const int kk = static_cast(u_mag / dur); + + bin_array[i] = ii + jj * n1 + kk * n1 * n2; + index_array[i] = i; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void labelOnCartesianVelocityGrid (const amrex::ParticleReal ux[], + const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], + const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop ) const + { + for (int i = cell_start; i < cell_stop; ++i) + { + const int ii = static_cast((ux[indices[i]] - ux_min) / dux); + const int jj = static_cast((uy[indices[i]] - uy_min) / duy); + const int kk = static_cast((uz[indices[i]] - uz_min) / duz); + + bin_array[i] = ii + jj * n1 + kk * n1 * n2; + index_array[i] = i; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator() (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop) const + { + if (velocity_grid_type == VelocityGridType::Spherical) { + labelOnSphericalVelocityGrid( + ux, uy, uz, indices, bin_array, index_array, cell_start, + cell_stop + ); + } + else if (velocity_grid_type == VelocityGridType::Cartesian) { + labelOnCartesianVelocityGrid( + ux, uy, uz, indices, bin_array, index_array, cell_start, + cell_stop + ); + } + } + + VelocityGridType velocity_grid_type; + int n1, n2; + amrex::ParticleReal dur, dutheta, duphi; + amrex::ParticleReal dux, duy, duz; + amrex::ParticleReal ux_min, uy_min, uz_min, ux_max, uy_max; + }; + +private: + VelocityGridType m_velocity_grid_type; + + int m_min_ppc = 1; + int m_ntheta, m_nphi; + amrex::ParticleReal m_delta_ur; + amrex::Vector m_delta_u; +}; +#endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp new file mode 100644 index 00000000000..db4ddb801c6 --- /dev/null +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -0,0 +1,315 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "VelocityCoincidenceThinning.H" + + +VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& species_name) +{ + using namespace amrex::literals; + + const amrex::ParmParse pp_species_name(species_name); + + utils::parser::queryWithParser( + pp_species_name, "resampling_min_ppc", m_min_ppc + ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_min_ppc >= 1, + "Resampling min_ppc should be greater than or equal to 1" + ); + + std::string velocity_grid_type_str = "spherical"; + pp_species_name.query( + "resampling_algorithm_velocity_grid_type", velocity_grid_type_str + ); + if (velocity_grid_type_str == "spherical") { + m_velocity_grid_type = VelocityGridType::Spherical; + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_theta", m_ntheta + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_phi", m_nphi + ); + } + else if (velocity_grid_type_str == "cartesian") { + m_velocity_grid_type = VelocityGridType::Cartesian; + utils::parser::getArrWithParser( + pp_species_name, "resampling_algorithm_delta_u", m_delta_u + ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_delta_u.size() == 3, + "resampling_algorithm_delta_u must have three components."); + } + else { + WARPX_ABORT_WITH_MESSAGE("Unkown velocity grid type."); + } +} + +void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, + WarpXParticleContainer * const pc) const +{ + using namespace amrex::literals; + + auto& ptile = pc->ParticlesAt(lev, pti); + const auto n_parts_in_tile = pti.numParticles(); + auto& soa = ptile.GetStructOfArrays(); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) + auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); +#elif defined(WARPX_DIM_RZ) + auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); // rename to PIdx::r after PR #4667 +#endif +#if defined(WARPX_DIM_3D) + auto * const AMREX_RESTRICT y = soa.GetRealData(PIdx::y).data(); +#endif + auto * const AMREX_RESTRICT z = soa.GetRealData(PIdx::z).data(); + auto * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); + auto * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); + auto * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); + auto * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); + auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + + // Using this function means that we must loop over the cells in the ParallelFor. + auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); + + const auto n_cells = static_cast(bins.numBins()); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); + + const auto min_ppc = m_min_ppc; + + const auto mass = pc->getMass(); + + // check if species mass > 0 + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass > 0, + "VelocityCoincidenceThinning does not yet work for massless particles." + ); + + // create a GPU vector to hold the momentum cluster index for each particle + amrex::Gpu::DeviceVector momentum_bin_number(n_parts_in_tile); + auto* momentum_bin_number_data = momentum_bin_number.data(); + + // create a GPU vector to hold the index sorting for the momentum bins + amrex::Gpu::DeviceVector sorted_indices(n_parts_in_tile); + auto* sorted_indices_data = sorted_indices.data(); + + constexpr auto c2 = PhysConst::c * PhysConst::c; + + auto velocityBinCalculator = VelocityBinCalculator(); + velocityBinCalculator.velocity_grid_type = m_velocity_grid_type; + if (m_velocity_grid_type == VelocityGridType::Spherical) { + velocityBinCalculator.dur = m_delta_ur; + velocityBinCalculator.n1 = m_ntheta; + velocityBinCalculator.n2 = m_nphi; + velocityBinCalculator.dutheta = 2.0_prt * MathConst::pi / m_ntheta; + velocityBinCalculator.duphi = MathConst::pi / m_nphi; + } + else if (m_velocity_grid_type == VelocityGridType::Cartesian) { + velocityBinCalculator.dux = m_delta_u[0]; + velocityBinCalculator.duy = m_delta_u[1]; + velocityBinCalculator.duz = m_delta_u[2]; + + // get the minimum and maximum velocities to determine the velocity space + // grid boundaries + { + using ReduceOpsT = amrex::TypeMultiplier; + using ReduceDataT = amrex::TypeMultiplier; + ReduceOpsT reduce_op; + ReduceDataT reduce_data(reduce_op); + using ReduceTuple = typename ReduceDataT::Type; + reduce_op.eval(n_parts_in_tile, reduce_data, [=] AMREX_GPU_DEVICE(int i) -> ReduceTuple { + return {ux[i], uy[i], uz[i], ux[i], uy[i]}; + }); + auto hv = reduce_data.value(reduce_op); + velocityBinCalculator.ux_min = amrex::get<0>(hv); + velocityBinCalculator.uy_min = amrex::get<1>(hv); + velocityBinCalculator.uz_min = amrex::get<2>(hv); + velocityBinCalculator.ux_max = amrex::get<3>(hv); + velocityBinCalculator.uy_max = amrex::get<4>(hv); + } + + velocityBinCalculator.n1 = static_cast( + std::ceil((velocityBinCalculator.ux_max - velocityBinCalculator.ux_min) / m_delta_u[0]) + ); + velocityBinCalculator.n2 = static_cast( + std::ceil((velocityBinCalculator.uy_max - velocityBinCalculator.uy_min) / m_delta_u[1]) + ); + } + auto heapSort = HeapSort(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles that are in the cell `i_cell` are + // given by the `indices[cell_start:cell_stop]` + const auto cell_start = static_cast(cell_offsets[i_cell]); + const auto cell_stop = static_cast(cell_offsets[i_cell+1]); + const auto cell_numparts = cell_stop - cell_start; + + // do nothing for cells with less particles than min_ppc + // (this intentionally includes skipping empty cells, too) + if (cell_numparts < min_ppc) { + return; + } + + // Loop over particles and label them with the appropriate momentum bin + // number. Also assign initial ordering to the sorted_indices array. + velocityBinCalculator( + ux, uy, uz, indices, momentum_bin_number_data, sorted_indices_data, + cell_start, cell_stop + ); + + // sort indices based on comparing values in momentum_bin_number + heapSort(sorted_indices_data, momentum_bin_number_data, cell_start, cell_numparts); + + // initialize variables used to hold cluster totals + int particles_in_bin = 0; + amrex::ParticleReal total_weight = 0._prt, total_energy = 0._prt; +#if !defined(WARPX_DIM_1D_Z) + amrex::ParticleReal cluster_x = 0._prt; +#endif +#if defined(WARPX_DIM_3D) + amrex::ParticleReal cluster_y = 0._prt; +#endif + amrex::ParticleReal cluster_z = 0._prt; + amrex::ParticleReal cluster_ux = 0._prt, cluster_uy = 0._prt, cluster_uz = 0._prt; + + // Finally, loop through the particles in the cell and merge + // ones in the same momentum bin + for (int i = cell_start; i < cell_stop; ++i) + { + particles_in_bin += 1; + const auto part_idx = indices[sorted_indices_data[i]]; + +#if !defined(WARPX_DIM_1D_Z) + cluster_x += w[part_idx]*x[part_idx]; +#endif +#if defined(WARPX_DIM_3D) + cluster_y += w[part_idx]*y[part_idx]; +#endif + cluster_z += w[part_idx]*z[part_idx]; + cluster_ux += w[part_idx]*ux[part_idx]; + cluster_uy += w[part_idx]*uy[part_idx]; + cluster_uz += w[part_idx]*uz[part_idx]; + total_weight += w[part_idx]; + total_energy += w[part_idx] * Algorithms::KineticEnergy( + ux[part_idx], uy[part_idx], uz[part_idx], mass + ); + + // check if this is the last particle in the current momentum bin + if ( + (i == cell_stop - 1) + || (momentum_bin_number_data[sorted_indices_data[i]] != momentum_bin_number_data[sorted_indices_data[i + 1]]) + ) { + // check if the bin has more than 2 particles in it + if ( particles_in_bin > 2 && total_weight > std::numeric_limits::min() ){ + // get average quantities for the bin +#if !defined(WARPX_DIM_1D_Z) + cluster_x /= total_weight; +#endif +#if defined(WARPX_DIM_3D) + cluster_y /= total_weight; +#endif + cluster_z /= total_weight; + cluster_ux /= total_weight; + cluster_uy /= total_weight; + cluster_uz /= total_weight; + + // perform merging of momentum bin particles + auto u_perp2 = cluster_ux*cluster_ux + cluster_uy*cluster_uy; + auto u_perp = std::sqrt(u_perp2); + auto cluster_u_mag2 = u_perp2 + cluster_uz*cluster_uz; + auto cluster_u_mag = std::sqrt(cluster_u_mag2); + + // calculate required velocity magnitude to achieve + // energy conservation + auto v_mag2 = total_energy / total_weight * ( + (total_energy / total_weight + 2._prt * mass * c2 ) + / (mass * mass * c2) + ); + auto v_perp = (v_mag2 > cluster_u_mag2) ? std::sqrt(v_mag2 - cluster_u_mag2) : 0_prt; + + // choose random angle for new velocity vector + auto phi = amrex::Random(engine) * MathConst::pi; + + // set new velocity components based on chosen phi + auto vx = v_perp * std::cos(phi); + auto vy = v_perp * std::sin(phi); + + // calculate rotation angles to parallel coord. frame + auto cos_theta = (cluster_u_mag > 0._prt) ? cluster_uz / cluster_u_mag : 0._prt; + auto sin_theta = (cluster_u_mag > 0._prt) ? u_perp / cluster_u_mag : 0._prt; + auto cos_phi = (u_perp > 0._prt) ? cluster_ux / u_perp : 0._prt; + auto sin_phi = (u_perp > 0._prt) ? cluster_uy / u_perp : 0._prt; + + // rotate new velocity vector to labframe + auto ux_new = ( + vx * cos_theta * cos_phi - vy * sin_phi + + cluster_u_mag * sin_theta * cos_phi + ); + auto uy_new = ( + vx * cos_theta * sin_phi + vy * cos_phi + + cluster_u_mag * sin_theta * sin_phi + ); + auto uz_new = -vx * sin_theta + cluster_u_mag * cos_theta; + + // set the last two particles' attributes according to + // the bin's aggregate values + const auto part_idx2 = indices[sorted_indices_data[i - 1]]; + + w[part_idx] = total_weight / 2._prt; + w[part_idx2] = total_weight / 2._prt; +#if !defined(WARPX_DIM_1D_Z) + x[part_idx] = cluster_x; + x[part_idx2] = cluster_x; +#endif +#if defined(WARPX_DIM_3D) + y[part_idx] = cluster_y; + y[part_idx2] = cluster_y; +#endif + z[part_idx] = cluster_z; + z[part_idx2] = cluster_z; + + ux[part_idx] = ux_new; + uy[part_idx] = uy_new; + uz[part_idx] = uz_new; + ux[part_idx2] = 2._prt * cluster_ux - ux_new; + uy[part_idx2] = 2._prt * cluster_uy - uy_new; + uz[part_idx2] = 2._prt * cluster_uz - uz_new; + + // set ids of merged particles so they will be removed + for (int j = 2; j < particles_in_bin; ++j){ + idcpu[indices[sorted_indices_data[i - j]]] = amrex::ParticleIdCpus::Invalid; + } + } + + // restart the tallies + particles_in_bin = 0; + total_weight = 0._prt; + total_energy = 0._prt; +#if !defined(WARPX_DIM_1D_Z) + cluster_x = 0_prt; +#endif +#if defined(WARPX_DIM_3D) + cluster_y = 0_prt; +#endif + cluster_z = 0_prt; + cluster_ux = 0_prt; + cluster_uy = 0_prt; + cluster_uz = 0_prt; + } + } + } + ); +} diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 7c56da457ed..e711cce15ff 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SHAPEFACTORS_H_ -#define SHAPEFACTORS_H_ +#ifndef WARPX_SHAPEFACTORS_H_ +#define WARPX_SHAPEFACTORS_H_ #include "Utils/TextMsg.H" @@ -243,4 +243,4 @@ struct Compute_shape_factor_pair } }; -#endif // SHAPEFACTORS_H_ +#endif // WARPX_SHAPEFACTORS_H_ diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7d2d5619da9..d4c325fbfb8 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -280,6 +280,7 @@ public: /// This is needed when solving Poisson's equation with periodic boundary conditions. /// amrex::ParticleReal sumParticleCharge(bool local = false); + amrex::ParticleReal sumParticleWeight(bool local = false); std::array meanParticleVelocity(bool local = false); @@ -321,6 +322,13 @@ public: int nattr_int, amrex::Vector> const & attr_int, int uniqueparticles, amrex::Long id=-1); + /** Remove particles with invalid ID + * + * This is a local operation (no MPI communication) and is thus preferable over `ReDistribute` + * (which also removes invalid particles, but involves MPI communications at the same time) + */ + void deleteInvalidParticles (); + virtual void ReadHeader (std::istream& is) = 0; virtual void WriteHeader (std::ostream& os) const = 0; @@ -418,6 +426,8 @@ public: virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} #endif + void setDoNotPush (bool flag) { do_not_push = flag; } + protected: int species_id; @@ -434,7 +444,7 @@ protected: //! instead of gathering fields from the finest patch level, gather from the coarsest bool m_gather_from_main_grid = false; - int do_not_push = 0; + bool do_not_push = false; int do_not_gather = 0; // Whether to allow particles outside of the simulation domain to be @@ -466,9 +476,9 @@ protected: //Species can receive a shared pointer to a QED engine (species for //which this is relevant should override these functions) virtual void - set_breit_wheeler_engine_ptr(std::shared_ptr){} + set_breit_wheeler_engine_ptr(const std::shared_ptr&){} virtual void - set_quantum_sync_engine_ptr(std::shared_ptr){} + set_quantum_sync_engine_ptr(const std::shared_ptr&){} int m_qed_breit_wheeler_ele_product; std::string m_qed_breit_wheeler_ele_product_name; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 6e6c0b27c85..8692dff3302 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -295,13 +295,29 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, ); } + // Move particles to their appropriate tiles + Redistribute(); + // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + deleteInvalidParticles(); #endif +} - Redistribute(); +void +WarpXParticleContainer::deleteInvalidParticles () { + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { + ParticleTileType& ptile = ParticlesAt(lev, pti); + removeInvalidParticles( ptile ); + } + } } /* \brief Current Deposition for thread thread_num @@ -461,8 +477,6 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, } WARPX_PROFILE_VAR_START(blp_deposit); - amrex::LayoutData * const costs = WarpX::getCosts(lev); - amrex::Real * const cost = costs ? &((*costs)[pti.index()]) : nullptr; // If doing shared mem current deposition, get tile info if (WarpX::do_shared_mem_current_deposition) { @@ -531,29 +545,29 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 4){ doDepositionSharedShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -567,29 +581,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) @@ -616,8 +626,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doChargeConservingDepositionShapeNImplicit<2>( xp_n_data, yp_n_data, zp_n_data, @@ -625,8 +634,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doChargeConservingDepositionShapeNImplicit<3>( xp_n_data, yp_n_data, zp_n_data, @@ -634,8 +642,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doChargeConservingDepositionShapeNImplicit<4>( xp_n_data, yp_n_data, zp_n_data, @@ -643,8 +650,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -673,8 +679,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVillasenorDepositionShapeNImplicit<2>( xp_n_data, yp_n_data, zp_n_data, @@ -682,8 +687,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVillasenorDepositionShapeNImplicit<3>( xp_n_data, yp_n_data, zp_n_data, @@ -691,8 +695,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVillasenorDepositionShapeNImplicit<4>( xp_n_data, yp_n_data, zp_n_data, @@ -700,8 +703,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else { @@ -716,29 +718,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVayDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else { // Direct deposition if (push_type == PushType::Explicit) { @@ -747,29 +745,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); @@ -782,8 +776,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeNImplicit<2>( GetPosition, wp.dataPtr() + offset, @@ -791,8 +784,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeNImplicit<3>( GetPosition, wp.dataPtr() + offset, @@ -800,8 +792,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeNImplicit<4>( GetPosition, wp.dataPtr() + offset, @@ -809,8 +800,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } } @@ -893,6 +883,12 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const long offset, const long np_to_deposit, const int thread_num, const int lev, const int depos_lev) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + rho->nComp() >= icomp - 1, + "Cannot deposit charge in rho component icomp=" + std::to_string(icomp) + + ": not enough components allocated (" + std::to_string(rho->nComp()) + "!" + ); + if (WarpX::do_shared_mem_charge_deposition) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || @@ -1084,8 +1080,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } WARPX_PROFILE_VAR_START(blp_ppc_chd); - amrex::LayoutData* costs = WarpX::getCosts(lev); - amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; const auto GetPosition = GetParticlePosition(pti, offset); const Geometry& geom = Geom(lev); @@ -1095,26 +1089,26 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 4){ doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } #ifndef AMREX_USE_GPU @@ -1158,10 +1152,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); - // pointer to costs data - amrex::LayoutData* costs = WarpX::getCosts(lev); - amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; - AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noy); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); @@ -1172,7 +1162,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, icomp, nc, - cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize + WarpX::do_device_synchronize ); } } @@ -1224,6 +1214,12 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, const bool apply_boundary_and_scale_volume, const int icomp) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + rho->nComp() >= icomp - 1, + "Cannot deposit charge in rho component icomp=" + std::to_string(icomp) + + ": not enough components allocated (" + std::to_string(rho->nComp()) + "!" + ); + // Reset the rho array if reset is True int const nc = WarpX::ncomps; if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } @@ -1306,9 +1302,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) return rho; } -amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { +amrex::ParticleReal WarpXParticleContainer::sumParticleWeight(bool local) { - amrex::ParticleReal total_charge = 0.0; + amrex::ParticleReal total_weight = 0.0; ReduceOps reduce_op; ReduceData reduce_data(reduce_op); @@ -1328,11 +1324,15 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { } } - total_charge = get<0>(reduce_data.value()); + total_weight = get<0>(reduce_data.value()); - if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } - total_charge *= this->charge; - return total_charge; + if (!local) { ParallelDescriptor::ReduceRealSum(total_weight); } + return total_weight; +} + +amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { + + return this->sumParticleWeight(local) * this->charge; } std::array WarpXParticleContainer::meanParticleVelocity(bool local) { @@ -1517,6 +1517,10 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) // without runtime component). void WarpXParticleContainer::defineAllParticleTiles () noexcept { + // Call the parent class's method + NamedComponentParticleContainer::defineAllParticleTiles(); + + // Resize the tmp_particle_data (no present in parent class) tmp_particle_data.resize(finestLevel()+1); for (int lev = 0; lev <= finestLevel(); ++lev) { @@ -1525,7 +1529,6 @@ void WarpXParticleContainer::defineAllParticleTiles () noexcept const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); tmp_particle_data[lev][std::make_pair(grid_id,tile_id)]; - DefineAndReturnParticleTile(lev, grid_id, tile_id); } } } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 07793a373f3..aa2cd7a2091 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -109,6 +109,10 @@ void init_WarpXParticleContainer (py::module& m) &WarpXParticleContainer::TotalNumberOfParticles, py::arg("valid_particles_only"), py::arg("local") ) + .def("sum_particle_weight", + &WarpXParticleContainer::sumParticleWeight, + py::arg("local") + ) .def("sum_particle_charge", &WarpXParticleContainer::sumParticleCharge, py::arg("local") @@ -133,5 +137,9 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("lev"), py::arg("local") ) + .def("set_do_not_push", + [](WarpXParticleContainer& pc, bool flag) { pc.setDoNotPush(flag); }, + py::arg("flag") + ) ; } diff --git a/Source/Python/callbacks.H b/Source/Python/callbacks.H index f0c98caf636..7ec78782bed 100644 --- a/Source/Python/callbacks.H +++ b/Source/Python/callbacks.H @@ -29,21 +29,21 @@ extern WARPX_EXPORT std::map< std::string, std::function > warpx_callbac /** * \brief Function to install the given name and function in warpx_callback_py_map */ -void InstallPythonCallback ( std::string name, std::function callback ); +void InstallPythonCallback ( const std::string& name, std::function callback ); /** * \brief Function to check if the given name is a key in warpx_callback_py_map */ -bool IsPythonCallbackInstalled ( std::string name ); +bool IsPythonCallbackInstalled ( const std::string& name ); /** * \brief Function to look for and execute Python callbacks */ -void ExecutePythonCallback ( std::string name ); +void ExecutePythonCallback ( const std::string& name ); /** * \brief Function to clear the given callback name from warpx_callback_py_map */ -void ClearPythonCallback ( std::string name ); +void ClearPythonCallback ( const std::string& name ); #endif // WARPX_PY_CALLBACKS_H_ diff --git a/Source/Python/callbacks.cpp b/Source/Python/callbacks.cpp index 930c88e65d1..79f15c62835 100644 --- a/Source/Python/callbacks.cpp +++ b/Source/Python/callbacks.cpp @@ -15,18 +15,18 @@ std::map< std::string, std::function > warpx_callback_py_map; -void InstallPythonCallback ( std::string name, std::function callback ) +void InstallPythonCallback ( const std::string& name, std::function callback ) { - warpx_callback_py_map[name] = callback; + warpx_callback_py_map[name] = std::move(callback); } -bool IsPythonCallbackInstalled ( std::string name ) +bool IsPythonCallbackInstalled ( const std::string& name ) { return (warpx_callback_py_map.count(name) == 1u); } // Execute Python callbacks of the type given by the input string -void ExecutePythonCallback ( std::string name ) +void ExecutePythonCallback ( const std::string& name ) { if ( IsPythonCallbackInstalled(name) ) { WARPX_PROFILE("warpx_py_" + name); @@ -47,7 +47,7 @@ void ExecutePythonCallback ( std::string name ) } } -void ClearPythonCallback ( std::string name ) +void ClearPythonCallback ( const std::string& name ) { warpx_callback_py_map.erase(name); } diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index c2da9577947..0339b766e38 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -164,7 +164,7 @@ amrex::Parser utils::parser::makeParser ( for (auto it = symbols.begin(); it != symbols.end(); ) { // Always parsing in double precision avoids potential overflows that may occur when parsing // user's expressions because of the limited range of exponentials in single precision - double v; + double v = 0.0; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( recursive_symbols.count(*it)==0, diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 735fc7993f1..c34516aedc5 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef UTILS_WARPXALGORITHMSELECTION_H_ -#define UTILS_WARPXALGORITHMSELECTION_H_ +#ifndef WARPX_UTILS_WARPXALGORITHMSELECTION_H_ +#define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include @@ -55,6 +55,12 @@ struct GridType { }; }; +enum struct PatchType +{ + fine, + coarse +}; + struct ElectromagneticSolverAlgo { enum { None = 0, @@ -75,6 +81,13 @@ struct ElectrostaticSolverAlgo { }; }; +struct PoissonSolverAlgo { + enum { + Multigrid = 1, + IntegratedGreenFunction = 2, + }; +}; + struct ParticlePusherAlgo { enum { Boris = 0, @@ -132,27 +145,27 @@ struct RhoInTime { struct LoadBalanceCostsUpdateAlgo { enum { Timers = 0, //!< load balance according to in-code timer-based weights (i.e., with `costs`) - Heuristic = 1, /**< load balance according to weights computed from number of cells - and number of particles per box (i.e., with `costs_heuristic`)*/ - GpuClock = 2 + Heuristic = 1 /**< load balance according to weights computed from number of cells + and number of particles per box (i.e., with `costs_heuristic`) */ }; }; /** Field boundary conditions at the domain boundary */ -struct FieldBoundaryType { - enum { +enum struct FieldBoundaryType { PML = 0, Periodic = 1, - PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 + PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 PMC = 3, //!< perfect magnetic conductor (PMC) with B_tangential=0 Damped = 4, // Fields in the guard cells are damped for PSATD - //in the moving window direction + //in the moving window direction Absorbing_SilverMueller = 5, // Silver-Mueller boundary condition Neumann = 6, // For electrostatic, the normal E is set to zero - None = 7 // The fields values at the boundary are not updated. This is + None = 7, // The fields values at the boundary are not updated. This is // useful for RZ simulations, at r=0. - }; + Open = 8 // Used in the Integrated Green Function Poisson solver + // Note that the solver implicitely assumes open BCs: + // no need to enforce them separately }; /** Particle boundary conditions at the domain boundary @@ -161,7 +174,8 @@ enum struct ParticleBoundaryType { Absorbing = 0, //!< particles crossing domain boundary are removed Open = 1, //!< particles cross domain boundary leave with damped j Reflecting = 2, //!< particles are reflected - Periodic = 3 + Periodic = 3, //!< particles are introduced from the periodic boundary + Thermal = 4 }; /** MPI reductions @@ -180,7 +194,7 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ); /** Select BC Type for fields, if field=true * else select BCType for particles. */ -int +FieldBoundaryType GetFieldBCTypeInteger( std::string BCType ); /** Select BC Type for particles. @@ -188,4 +202,9 @@ GetFieldBCTypeInteger( std::string BCType ); ParticleBoundaryType GetParticleBCTypeInteger( std::string BCType ); -#endif // UTILS_WARPXALGORITHMSELECTION_H_ +/** Find the name associated with a BC type + */ +std::string +GetFieldBCTypeString( FieldBoundaryType fb_type ); + +#endif // WARPX_UTILS_WARPXALGORITHMSELECTION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index abaf17f0a2c..b1f93c2f6c8 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -55,6 +55,12 @@ const std::map electrostatic_solver_algo_to_int = { {"default", ElectrostaticSolverAlgo::None } }; +const std::map poisson_solver_algo_to_int = { + {"multigrid", PoissonSolverAlgo::Multigrid}, + {"fft", PoissonSolverAlgo::IntegratedGreenFunction}, + {"default", PoissonSolverAlgo::Multigrid } +}; + const std::map particle_pusher_algo_to_int = { {"boris", ParticlePusherAlgo::Boris }, {"vay", ParticlePusherAlgo::Vay }, @@ -101,7 +107,6 @@ const std::map rho_in_time_to_int = { const std::map load_balance_costs_update_algo_to_int = { {"timers", LoadBalanceCostsUpdateAlgo::Timers }, - {"gpuclock", LoadBalanceCostsUpdateAlgo::GpuClock }, {"heuristic", LoadBalanceCostsUpdateAlgo::Heuristic }, {"default", LoadBalanceCostsUpdateAlgo::Timers } }; @@ -118,7 +123,7 @@ const std::map MacroscopicSolver_algo_to_int = { {"default", MacroscopicSolverAlgo::BackwardEuler} }; -const std::map FieldBCType_algo_to_int = { +const std::map FieldBCType_algo_to_enum = { {"pml", FieldBoundaryType::PML}, {"periodic", FieldBoundaryType::Periodic}, {"pec", FieldBoundaryType::PEC}, @@ -126,6 +131,7 @@ const std::map FieldBCType_algo_to_int = { {"damped", FieldBoundaryType::Damped}, {"absorbing_silver_mueller", FieldBoundaryType::Absorbing_SilverMueller}, {"neumann", FieldBoundaryType::Neumann}, + {"open", FieldBoundaryType::Open}, {"none", FieldBoundaryType::None}, {"default", FieldBoundaryType::PML} }; @@ -135,6 +141,7 @@ const std::map ParticleBCType_algo_to_enum = {"open", ParticleBoundaryType::Open}, {"reflecting", ParticleBoundaryType::Reflecting}, {"periodic", ParticleBoundaryType::Periodic}, + {"thermal", ParticleBoundaryType::Thermal}, {"default", ParticleBoundaryType::Absorbing} }; @@ -163,6 +170,8 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ algo_to_int = grid_to_int; } else if (0 == std::strcmp(pp_search_key, "do_electrostatic")) { algo_to_int = electrostatic_solver_algo_to_int; + } else if (0 == std::strcmp(pp_search_key, "poisson_solver")) { + algo_to_int = poisson_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "particle_pusher")) { algo_to_int = particle_pusher_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "current_deposition")) { @@ -213,21 +222,21 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ return algo_to_int[algo]; } -int +FieldBoundaryType GetFieldBCTypeInteger( std::string BCType ){ std::transform(BCType.begin(), BCType.end(), BCType.begin(), ::tolower); - if (FieldBCType_algo_to_int.count(BCType) == 0) { + if (FieldBCType_algo_to_enum.count(BCType) == 0) { std::string error_message = "Invalid string for field/particle BC. : " + BCType + "\nThe valid values are : \n"; - for (const auto &valid_pair : FieldBCType_algo_to_int) { + for (const auto &valid_pair : FieldBCType_algo_to_enum) { if (valid_pair.first != "default"){ error_message += " - " + valid_pair.first + "\n"; } } WARPX_ABORT_WITH_MESSAGE(error_message); } - // return FieldBCType_algo_to_int[BCType]; // This operator cannot be used for a const map - return FieldBCType_algo_to_int.at(BCType); + // return FieldBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map + return FieldBCType_algo_to_enum.at(BCType); } ParticleBoundaryType @@ -246,3 +255,15 @@ GetParticleBCTypeInteger( std::string BCType ){ // return ParticleBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map return ParticleBCType_algo_to_enum.at(BCType); } + +std::string +GetFieldBCTypeString( FieldBoundaryType fb_type ) { + std::string boundary_name; + for (const auto &valid_pair : FieldBCType_algo_to_enum) { + if ((valid_pair.second == fb_type)&&(valid_pair.first != "default")){ + boundary_name = valid_pair.first; + break; + } + } + return boundary_name; +} diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index ad3194edfc3..1de03eb61f0 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -64,7 +64,7 @@ namespace WarpXUtilIO{ * @param[in] data Vector containing binary data to write on disk * return true if it succeeds, false otherwise */ -bool WriteBinaryDataOnFile(std::string filename, const amrex::Vector& data); +bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data); } @@ -116,7 +116,7 @@ namespace WarpXUtilLoadBalance * @param[in] dm the dmap to check * @return consistent whether the grids are consistent or not. */ - bool doCosts (const amrex::LayoutData* cost, amrex::BoxArray ba, + bool doCosts (const amrex::LayoutData* cost, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); } diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 5b917a81fdf..d2691685c53 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -266,7 +266,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) } namespace WarpXUtilIO{ - bool WriteBinaryDataOnFile(std::string filename, const amrex::Vector& data) + bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data) { std::ofstream of{filename, std::ios::binary}; of.write(data.data(), data.size()); @@ -404,6 +404,7 @@ void ReadBCParams () const ParmParse pp_warpx("warpx"); const ParmParse pp_algo("algo"); const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); + const int poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); if (pp_geometry.queryarr("is_periodic", geom_periodicity)) { @@ -469,6 +470,14 @@ void ReadBCParams () ), "PEC boundary not implemented for PSATD, yet!" ); + + if(WarpX::field_boundary_lo[idim] == FieldBoundaryType::Open && + WarpX::field_boundary_hi[idim] == FieldBoundaryType::Open){ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction, + "Field open boundary conditions are only implemented for the FFT-based Poisson solver" + ); + } } // Appending periodicity information to input so that it can be used by amrex @@ -481,11 +490,11 @@ void ReadBCParams () namespace WarpXUtilLoadBalance { - bool doCosts (const amrex::LayoutData* costs, const amrex::BoxArray ba, + bool doCosts (const amrex::LayoutData* cost, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) { - const bool consistent = costs && (dm == costs->DistributionMap()) && - (ba.CellEqual(costs->boxArray())) && + const bool consistent = cost && (dm == cost->DistributionMap()) && + (ba.CellEqual(cost->boxArray())) && (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers); return consistent; } diff --git a/Source/WarpX.H b/Source/WarpX.H index 79352274068..8ad7820e4cd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -38,6 +38,7 @@ #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" @@ -69,26 +70,17 @@ #include #include #include +#include #include #include #include #include -#include - - -enum struct PatchType : int -{ - fine, - coarse -}; class WARPX_EXPORT WarpX : public amrex::AmrCore { public: - friend class PML; - static WarpX& GetInstance (); static void ResetInstance (); @@ -131,6 +123,7 @@ public: MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } + [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} #ifdef AMREX_USE_EB amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} @@ -175,7 +168,7 @@ public: //! Flags whether the Picard iterations are required to converge static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy - * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). + * being used (0 or 1 corresponding to timers or heuristic). */ static short load_balance_costs_update_algo; //! Integer that corresponds to electromagnetic Maxwell solver (vacuum - 0, macroscopic - 1) @@ -192,20 +185,20 @@ public: * lower domain boundaries * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) */ - static amrex::Vector field_boundary_lo; + static amrex::Vector field_boundary_lo; /** Integers that correspond to boundary condition applied to fields at the * upper domain boundaries * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) */ - static amrex::Vector field_boundary_hi; + static amrex::Vector field_boundary_hi; /** Integers that correspond to boundary condition applied to particles at the * lower domain boundaries - * (0 to 3 correspond to Absorbing, Open, Reflecting, Periodic) + * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ static amrex::Vector particle_boundary_lo; /** Integers that correspond to boundary condition applied to particles at the * upper domain boundaries - * (0 to 3 correspond to Absorbing, Open, Reflecting, Periodic) + * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ static amrex::Vector particle_boundary_hi; @@ -222,7 +215,7 @@ public: //! If true, the current is deposited on a nodal grid and then centered onto a staggered grid //! using finite centering of order given by #current_centering_nox, #current_centering_noy, //! and #current_centering_noz - static bool do_current_centering; + bool do_current_centering = false; //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation @@ -466,72 +459,56 @@ public: static std::map multifab_map; static std::map imultifab_map; - [[nodiscard]] std::array - get_array_Bfield_aux (const int lev) const { - return { - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get() - }; - } + /** + * \brief + * Check if a field is initialized. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return true if the field is initialized, false otherwise + */ + [[nodiscard]] bool + isFieldInitialized (warpx::fields::FieldType field_type, int lev, int direction = 0) const; + /** + * \brief + * Get a pointer to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointer (warpx::fields::FieldType field_type, int lev, int direction = 0) const; + + /** + * \brief + * For vector fields, get an array of three pointers to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * + * \return an array of three pointers amrex::MultiFab* containing the field data + */ [[nodiscard]] std::array - get_array_Efield_aux (const int lev) const { - return { - Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get() - }; - } + getFieldPointerArray (warpx::fields::FieldType field_type, int lev) const; - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_aux (int lev, int direction) const { return Efield_aux[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_aux (int lev, int direction) const { return Bfield_aux[lev][direction].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_fp (int lev, int direction) const { return Efield_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_fp (int lev, int direction) const { return Bfield_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_fp (int lev, int direction) const { return current_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_fp_nodal (int lev, int direction) const { return current_fp_nodal[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_rho_fp (int lev) const { return rho_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_F_fp (int lev) const { return F_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_G_fp (int lev) const { return G_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_phi_fp (int lev) const { return phi_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_vector_potential_fp (int lev, int direction) const { return vector_potential_fp_nodal[lev][direction].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_cp (int lev, int direction) const { return Efield_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_cp (int lev, int direction) const { return Bfield_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_cp (int lev, int direction) const { return current_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_rho_cp (int lev) const { return rho_cp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_F_cp (int lev) const { return F_cp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_G_cp (int lev) const { return G_cp[lev].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_edge_lengths (int lev, int direction) const { return m_edge_lengths[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_face_areas (int lev, int direction) const { return m_face_areas[lev][direction].get(); } - - const amrex::MultiFab& getEfield (int lev, int direction) {return *Efield_aux[lev][direction];} - const amrex::MultiFab& getBfield (int lev, int direction) {return *Bfield_aux[lev][direction];} - - const amrex::MultiFab& getcurrent_cp (int lev, int direction) {return *current_cp[lev][direction];} - const amrex::MultiFab& getEfield_cp (int lev, int direction) {return *Efield_cp[lev][direction];} - const amrex::MultiFab& getBfield_cp (int lev, int direction) {return *Bfield_cp[lev][direction];} - const amrex::MultiFab& getrho_cp (int lev) {return *rho_cp[lev];} - const amrex::MultiFab& getF_cp (int lev) {return *F_cp[lev];} - const amrex::MultiFab& getG_cp (int lev) {return *G_cp[lev];} - - const amrex::MultiFab& getcurrent_fp (int lev, int direction) {return *current_fp[lev][direction];} - const amrex::MultiFab& getEfield_fp (int lev, int direction) {return *Efield_fp[lev][direction];} - const amrex::MultiFab& getBfield_fp (int lev, int direction) {return *Bfield_fp[lev][direction];} - const amrex::MultiFab& getrho_fp (int lev) {return *rho_fp[lev];} - const amrex::MultiFab& getphi_fp (int lev) {return *phi_fp[lev];} - const amrex::MultiFab& getF_fp (int lev) {return *F_fp[lev];} - const amrex::MultiFab& getG_fp (int lev) {return *G_fp[lev];} - - const amrex::MultiFab& getEfield_avg_fp (int lev, int direction) {return *Efield_avg_fp[lev][direction];} - const amrex::MultiFab& getBfield_avg_fp (int lev, int direction) {return *Bfield_avg_fp[lev][direction];} - const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];} - const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} - - const amrex::MultiFab& getedgelengths (int lev, int direction) {return *m_edge_lengths[lev][direction];} - const amrex::MultiFab& getfaceareas (int lev, int direction) {return *m_face_areas[lev][direction];} + /** + * \brief + * Get a constant reference to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return a constant refernce to an amrex::MultiFab containing the field data + */ + [[nodiscard]] const amrex::MultiFab& + getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -667,13 +644,26 @@ public: static amrex::Real quantum_xi_c2; + /** Check and potentially compute load balancing + */ + void CheckLoadBalance (int step); + /** \brief perform load balance; compute and communicate new `amrex::DistributionMapping` */ void LoadBalance (); + /** \brief resets costs to zero */ void ResetCosts (); + /** Perform running average of the LB costs + * + * Only needed for timers cost update, heuristic load balance considers the + * instantaneous costs. + * This gives more importance to most recent costs. + */ + void RescaleCosts (int step); + /** \brief returns the load balance interval */ [[nodiscard]] utils::parser::IntervalsParser get_load_balance_intervals () const @@ -762,6 +752,8 @@ public: void CopyJPML (); bool isAnyBoundaryPML(); + /** True if any of the particle boundary condition type is Thermal */ + static bool isAnyParticleBoundaryThermal(); PML* GetPML (int lev); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -912,6 +904,7 @@ public: static const amrex::iMultiFab* GatherBufferMasks (int lev); static int electrostatic_solver_id; + static int poisson_solver_id; // Parameters for lab frame electrostatic static amrex::Real self_fields_required_precision; @@ -963,6 +956,7 @@ public: */ [[nodiscard]] amrex::IntVect get_numprocs() const {return numprocs;} + bool m_boundary_potential_specified = false; ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler; void ComputeSpaceChargeField (bool reset_fields); void AddBoundaryField (); @@ -1042,8 +1036,8 @@ public: * for a specific field (specified by `F_name`) */ void ReadExternalFieldFromFile ( - std::string read_fields_from_path, amrex::MultiFab* mf, - std::string F_name, std::string F_component); + const std::string& read_fields_from_path, amrex::MultiFab* mf, + const std::string& F_name, const std::string& F_component); /** * \brief @@ -1578,6 +1572,20 @@ private: amrex::Vector, 3 > > current_buf; amrex::Vector > charge_buf; + /** + * \brief + * Get a pointer to the field data. Does not check if the pointer + * is not nullptr. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; + // PML int do_pml = 0; int do_silver_mueller = 0; @@ -1586,7 +1594,7 @@ private: int pml_has_particles = 0; int do_pml_j_damping = 0; int do_pml_in_domain = 0; - static int do_similar_dm_pml; + bool do_similar_dm_pml = true; bool do_pml_dive_cleaning; // default set in WarpX.cpp bool do_pml_divb_cleaning; // default set in WarpX.cpp amrex::Vector do_pml_Lo; @@ -1730,8 +1738,41 @@ private: return *m_field_factory[lev]; } + /** Stop the simulation at the end of the current step due to a received Unix signal? + */ + bool m_exit_loop_due_to_interrupt_signal = false; + + /** Stop the simulation at the end of the current step? + */ + [[nodiscard]] + bool checkStopSimulation (amrex::Real cur_time); + + /** Print Unused Parameter Warnings after Step 1 + * + * Instead of waiting for a simulation to end, we already do an early "unused parameter check" + * after step 1 to inform users early of potential issues with their simulation setup. + */ + void checkEarlyUnusedParams (); + + /** Perform essential particle house keeping at boundaries + * + * Inject, communicate, scrape and sort particles. + * + * @param step current step + * @param cur_time current time + * @param num_moved number of cells the moving window moved + */ + void HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved); + void ScrapeParticles (); + /** Update the E and B fields in the explicit em PIC scheme. + * + * At the beginning, we have B^{n} and E^{n}. + * Particles have p^{n} and x^{n}. + */ + void ExplicitFillBoundaryEBUpdateAux (); + void PushPSATD (); #ifdef WARPX_USE_PSATD diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index daed6a52fb4..d724ecd7571 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -85,6 +85,7 @@ #include using namespace amrex; +using namespace warpx::fields; int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; @@ -139,13 +140,11 @@ amrex::IntVect WarpX::shared_tilesize(AMREX_D_DECL(1,1,1)); #endif int WarpX::shared_mem_current_tpb = 128; -amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,0); -amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,0); +amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,FieldBoundaryType::PML); +amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,FieldBoundaryType::PML); amrex::Vector WarpX::particle_boundary_lo(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); amrex::Vector WarpX::particle_boundary_hi(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); -bool WarpX::do_current_centering = false; - int WarpX::n_rz_azimuthal_modes = 1; int WarpX::ncomps = 1; @@ -192,6 +191,7 @@ amrex::IntVect WarpX::sort_idx_type(AMREX_D_DECL(0,0,0)); bool WarpX::do_dynamic_scheduling = true; int WarpX::electrostatic_solver_id; +int WarpX::poisson_solver_id; Real WarpX::self_fields_required_precision = 1.e-11_rt; Real WarpX::self_fields_absolute_tolerance = 0.0_rt; int WarpX::self_fields_max_iters = 200; @@ -213,8 +213,6 @@ int WarpX::n_current_deposition_buffer = -1; short WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; -int WarpX::do_similar_dm_pml = 1; - #ifdef AMREX_USE_GPU bool WarpX::do_device_synchronize = true; #else @@ -765,15 +763,57 @@ WarpX::ReadParameters () pp_warpx, "self_fields_max_iters", self_fields_max_iters); pp_warpx.query("self_fields_verbosity", self_fields_verbosity); } + + poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); +#ifndef WARPX_DIM_3D + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, + "The FFT Poisson solver only works in 3D."); +#endif +#ifndef WARPX_USE_PSATD + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, + "To use the FFT Poisson solver, compile with WARPX_USE_PSATD=ON."); +#endif + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + ( + electrostatic_solver_id!=ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic || + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction + ), + "The FFT Poisson solver is not implemented in labframe-electromagnetostatic mode yet." + ); + // Parse the input file for domain boundary potentials const ParmParse pp_boundary("boundary"); - pp_boundary.query("potential_lo_x", m_poisson_boundary_handler.potential_xlo_str); - pp_boundary.query("potential_hi_x", m_poisson_boundary_handler.potential_xhi_str); - pp_boundary.query("potential_lo_y", m_poisson_boundary_handler.potential_ylo_str); - pp_boundary.query("potential_hi_y", m_poisson_boundary_handler.potential_yhi_str); - pp_boundary.query("potential_lo_z", m_poisson_boundary_handler.potential_zlo_str); - pp_boundary.query("potential_hi_z", m_poisson_boundary_handler.potential_zhi_str); - pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); + bool potential_specified = false; + // When reading the potential at the boundary from the input file, set this flag to true if any of the potential is specified + potential_specified |= pp_boundary.query("potential_lo_x", m_poisson_boundary_handler.potential_xlo_str); + potential_specified |= pp_boundary.query("potential_hi_x", m_poisson_boundary_handler.potential_xhi_str); + potential_specified |= pp_boundary.query("potential_lo_y", m_poisson_boundary_handler.potential_ylo_str); + potential_specified |= pp_boundary.query("potential_hi_y", m_poisson_boundary_handler.potential_yhi_str); + potential_specified |= pp_boundary.query("potential_lo_z", m_poisson_boundary_handler.potential_zlo_str); + potential_specified |= pp_boundary.query("potential_hi_z", m_poisson_boundary_handler.potential_zhi_str); +#if defined(AMREX_USE_EB) + potential_specified |= pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); +#endif + m_boundary_potential_specified = potential_specified; + if (potential_specified & (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC)) { + ablastr::warn_manager::WMRecordWarning( + "Algorithms", + "The input script specifies the electric potential (phi) at the boundary, but \ + also uses the hybrid PIC solver based on Ohm’s law. When using this solver, the \ + electric potential does not have any impact on the simulation.", + ablastr::warn_manager::WarnPriority::low); + } + else if (potential_specified & (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None)) { + ablastr::warn_manager::WMRecordWarning( + "Algorithms", + "The input script specifies the electric potential (phi) at the boundary so \ + an initial Poisson solve will be performed.", + ablastr::warn_manager::WarnPriority::low); + } + m_poisson_boundary_handler.buildParsers(); #ifdef WARPX_DIM_RZ pp_boundary.query("verboncoeur_axis_correction", verboncoeur_axis_correction); @@ -1313,10 +1353,6 @@ WarpX::ReadParameters () utils::parser::queryWithParser( pp_algo, "costs_heuristic_particles_wt", costs_heuristic_particles_wt); } -# ifndef WARPX_USE_GPUCLOCK - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(WarpX::load_balance_costs_update_algo!=LoadBalanceCostsUpdateAlgo::GpuClock, - "`algo.load_balance_costs_update = gpuclock` requires to compile with `-DWarpX_GPUCLOCK=ON`."); -# endif // !WARPX_USE_GPUCLOCK // Parse algo.particle_shape and check that input is acceptable // (do this only if there is at least one particle or laser species) @@ -3274,6 +3310,16 @@ WarpX::isAnyBoundaryPML() return false; } +bool +WarpX::isAnyParticleBoundaryThermal () +{ + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) {return true;} + if (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) {return true;} + } + return false; +} + std::string TagWithLevelSuffix (std::string name, int const level) { @@ -3357,3 +3403,127 @@ WarpX::AllocInitMultiFabFromModel ( } multifab_map[name_with_suffix] = mf.get(); } + +amrex::MultiFab* +WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const +{ + // This function does *not* check if the returned field pointer is != nullptr + + amrex::MultiFab* field_pointer = nullptr; + + switch(field_type) + { + case FieldType::Efield_aux : + field_pointer = Efield_aux[lev][direction].get(); + break; + case FieldType::Bfield_aux : + field_pointer = Bfield_aux[lev][direction].get(); + break; + case FieldType::Efield_fp : + field_pointer = Efield_fp[lev][direction].get(); + break; + case FieldType::Bfield_fp : + field_pointer = Bfield_fp[lev][direction].get(); + break; + case FieldType::current_fp : + field_pointer = current_fp[lev][direction].get(); + break; + case FieldType::current_fp_nodal : + field_pointer = current_fp_nodal[lev][direction].get(); + break; + case FieldType::rho_fp : + field_pointer = rho_fp[lev].get(); + break; + case FieldType::F_fp : + field_pointer = F_fp[lev].get(); + break; + case FieldType::G_fp : + field_pointer = G_fp[lev].get(); + break; + case FieldType::phi_fp : + field_pointer = phi_fp[lev].get(); + break; + case FieldType::vector_potential_fp : + field_pointer = vector_potential_fp_nodal[lev][direction].get(); + break; + case FieldType::Efield_cp : + field_pointer = Efield_cp[lev][direction].get(); + break; + case FieldType::Bfield_cp : + field_pointer = Bfield_cp[lev][direction].get(); + break; + case FieldType::current_cp : + field_pointer = current_cp[lev][direction].get(); + break; + case FieldType::rho_cp : + field_pointer = rho_cp[lev].get(); + break; + case FieldType::F_cp : + field_pointer = F_cp[lev].get(); + break; + case FieldType::G_cp : + field_pointer = G_cp[lev].get(); + break; + case FieldType::edge_lengths : + field_pointer = m_edge_lengths[lev][direction].get(); + break; + case FieldType::face_areas : + field_pointer = m_face_areas[lev][direction].get(); + break; + case FieldType::Efield_avg_fp : + field_pointer = Efield_avg_fp[lev][direction].get(); + break; + case FieldType::Bfield_avg_fp : + field_pointer = Bfield_avg_fp[lev][direction].get(); + break; + case FieldType::Efield_avg_cp : + field_pointer = Efield_avg_cp[lev][direction].get(); + break; + case FieldType::Bfield_avg_cp : + field_pointer = Bfield_avg_cp[lev][direction].get(); + break; + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + break; + } + + return field_pointer; +} + +bool +WarpX::isFieldInitialized (const FieldType field_type, const int lev, const int direction) const +{ + const bool is_field_init = (getFieldPointerUnchecked(field_type, lev, direction) != nullptr); + return is_field_init; +} + +amrex::MultiFab* +WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const +{ + auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_pointer != nullptr, "Requested field is not initialized!"); + return field_pointer; +} + +std::array +WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (field_type == FieldType::Efield_aux) || (field_type == FieldType::Bfield_aux) || + (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || + (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || + (field_type == FieldType::Efield_cp) || (field_type == FieldType::Bfield_cp) || + (field_type == FieldType::current_cp), "Requested field type is not a vector."); + + return std::array{ + getFieldPointer(field_type, lev, 0), + getFieldPointer(field_type, lev, 1), + getFieldPointer(field_type, lev, 2)}; +} + +const amrex::MultiFab& +WarpX::getField(FieldType field_type, const int lev, const int direction) const +{ + return *getFieldPointer(field_type, lev, direction); +} diff --git a/Source/ablastr/CMakeLists.txt b/Source/ablastr/CMakeLists.txt index a41fa4ae3a0..03b876fa1ab 100644 --- a/Source/ablastr/CMakeLists.txt +++ b/Source/ablastr/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(coarsen) add_subdirectory(math) -#add_subdirectory(fields) +add_subdirectory(fields) add_subdirectory(parallelization) #add_subdirectory(particles) #add_subdirectory(profiler) diff --git a/Source/ablastr/Make.package b/Source/ablastr/Make.package index 3426020e45d..b9ff3c72560 100644 --- a/Source/ablastr/Make.package +++ b/Source/ablastr/Make.package @@ -2,6 +2,7 @@ include $(WARPX_HOME)/Source/ablastr/coarsen/Make.package include $(WARPX_HOME)/Source/ablastr/math/Make.package +include $(WARPX_HOME)/Source/ablastr/fields/Make.package include $(WARPX_HOME)/Source/ablastr/parallelization/Make.package include $(WARPX_HOME)/Source/ablastr/particles/Make.package include $(WARPX_HOME)/Source/ablastr/utils/Make.package diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index b5661037c7e..51dcfc7dd08 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -41,12 +41,12 @@ namespace ablastr::coarsen::sample const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) { + if ( crse_ratio.allGT(amrex::IntVect(1)) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); } - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect().allGE(stag_dst-stag_src+ngrowvect), "source fine MultiFab does not have enough guard cells for this interpolation" ); // Auxiliary integer arrays (always 3D) diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt new file mode 100644 index 00000000000..f7dfb6763ec --- /dev/null +++ b/Source/ablastr/fields/CMakeLists.txt @@ -0,0 +1,9 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + if(WarpX_PSATD AND D EQUAL 3) + target_sources(ablastr_${SD} + PRIVATE + IntegratedGreenFunctionSolver.cpp + ) + endif() +endforeach() diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H new file mode 100644 index 00000000000..97ffdb5ac36 --- /dev/null +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H @@ -0,0 +1,66 @@ +/* Copyright 2019-2024 Arianna Formenti, Remi Lehe + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef ABLASTR_IGF_SOLVER_H +#define ABLASTR_IGF_SOLVER_H + +#include +#include +#include +#include +#include + + +#include +#include + + +namespace ablastr::fields +{ + + /** @brief Implements equation 2 in https://doi.org/10.1103/PhysRevSTAB.10.129901 + * with some modification to symmetrize the function. + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * @param[in] z z-coordinate of given location + * + * @return the integrated Green function G + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + IntegratedPotential (amrex::Real x, amrex::Real y, amrex::Real z) + { + using namespace amrex::literals; + + amrex::Real const r = std::sqrt( x*x + y*y + z*z ); + amrex::Real const G = + - 0.5_rt * z*z * std::atan( x*y/(z*r) ) + - 0.5_rt * y*y * std::atan( x*z/(y*r) ) + - 0.5_rt * x*x * std::atan( y*z/(x*r) ) + + y*z*std::asinh( x/std::sqrt(y*y + z*z) ) + + x*z*std::asinh( y/std::sqrt(x*x + z*z) ) + + x*y*std::asinh( z/std::sqrt(x*x + y*y) ); + return G; + } + + /** @brief Compute the electrostatic potential using the Integrated Green Function method + * as in http://dx.doi.org/10.1103/PhysRevSTAB.9.044204 + * + * @param[in] rho the charge density amrex::MultiFab + * @param[out] phi the electrostatic potential amrex::MultiFab + * @param[in] cell_size an arreay of 3 reals dx dy dz + * @param[in] ba amrex::BoxArray with the grid of a given level + */ + void + computePhiIGF (amrex::MultiFab const & rho, + amrex::MultiFab & phi, + std::array const & cell_size, + amrex::BoxArray const & ba); + +} // namespace ablastr::fields + +#endif // ABLASTR_IGF_SOLVER_H diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp new file mode 100644 index 00000000000..0767ecfb2f3 --- /dev/null +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -0,0 +1,196 @@ +/* Copyright 2019-2024 Arianna Formenti, Remi Lehe + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ +#include "IntegratedGreenFunctionSolver.H" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace ablastr::fields { + +void +computePhiIGF ( amrex::MultiFab const & rho, + amrex::MultiFab & phi, + std::array const & cell_size, + amrex::BoxArray const & ba ) +{ + using namespace amrex::literals; + + // Define box that encompasses the full domain + amrex::Box domain = ba.minimalBox(); + domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal + domain.grow( phi.nGrowVect() ); // include guard cells + + int const nx = domain.length(0); + int const ny = domain.length(1); + int const nz = domain.length(2); + + // Allocate 2x wider arrays for the convolution of rho with the Green function + // This also defines the box arrays for the global FFT: contains only one box; + amrex::Box const realspace_box = amrex::Box( + {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, + {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, + amrex::IntVect::TheNodeVector() ); + amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); + amrex::Box const spectralspace_box = amrex::Box( + {0,0,0}, + {nx, 2*ny-1, 2*nz-1}, + amrex::IntVect::TheNodeVector() ); + amrex::BoxArray const spectralspace_ba = amrex::BoxArray( spectralspace_box ); + // Define a distribution mapping for the global FFT, with only one box + amrex::DistributionMapping dm_global_fft; + dm_global_fft.define( realspace_ba ); + // Allocate required arrays + amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); + tmp_rho.setVal(0); + amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); + tmp_G.setVal(0); + // Allocate corresponding arrays in Fourier space + using SpectralField = amrex::FabArray< amrex::BaseFab< amrex::GpuComplex< amrex::Real > > >; + SpectralField tmp_rho_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); + SpectralField tmp_G_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); + + // Copy from rho to tmp_rho + tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); + + // Compute the integrated Green function + { + BL_PROFILE("Initialize Green function"); + amrex::BoxArray const domain_ba = amrex::BoxArray( domain ); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(domain_ba, dm_global_fft,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + amrex::Box const bx = mfi.tilebox(); + + amrex::IntVect const lo = realspace_box.smallEnd(); + amrex::IntVect const hi = realspace_box.bigEnd(); + + // Fill values of the Green function + amrex::Real const dx = cell_size[0]; + amrex::Real const dy = cell_size[1]; + amrex::Real const dz = cell_size[2]; + amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); + amrex::ParallelFor( bx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + int const k0 = k - lo[2]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::Real const z = k0*dz; + + amrex::Real const G_value = 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( + IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + ); + + tmp_G_arr(i,j,k) = G_value; + // Fill the rest of the array by periodicity + if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} + if (j0>0) {tmp_G_arr(i , hi[1]+1-j0, k ) = G_value;} + if (k0>0) {tmp_G_arr(i , j , hi[2]+1-k0) = G_value;} + if ((i0>0)&&(j0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, k ) = G_value;} + if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} + if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} + if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} + } + ); + } + } + + // Perform forward FFTs + auto forward_plan_rho = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + auto forward_plan_G = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + // Loop over boxes perform FFTs + for ( amrex::MFIter mfi(realspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + + // Note: the size of the real-space box and spectral-space box + // differ when using real-to-complex FFT. When initializing + // the FFT plan, the valid dimensions are those of the real-space box. + const amrex::IntVect fft_size = realspace_ba[mfi].length(); + + // FFT of rho + forward_plan_rho[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_rho[mfi].dataPtr(), + reinterpret_cast(tmp_rho_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(forward_plan_rho[mfi]); + + // FFT of G + forward_plan_G[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[mfi].dataPtr(), + reinterpret_cast(tmp_G_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(forward_plan_G[mfi]); + + } + + // Multiply tmp_G_fft and tmp_rho_fft in spectral space + // Store the result in-place in Gtmp_G_fft, to save memory + amrex::Multiply( tmp_G_fft, tmp_rho_fft, 0, 0, 1, 0); + + // Perform inverse FFT + auto backward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + // Loop over boxes perform FFTs + for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + + // Note: the size of the real-space box and spectral-space box + // differ when using real-to-complex FFT. When initializing + // the FFT plan, the valid dimensions are those of the real-space box. + const amrex::IntVect fft_size = realspace_ba[mfi].length(); + + // Inverse FFT: is done in-place, in the array of G + backward_plan[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[mfi].dataPtr(), + reinterpret_cast( tmp_G_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(backward_plan[mfi]); + } + // Normalize, since (FFT + inverse FFT) results in a factor N + const amrex::Real normalization = 1._rt / realspace_box.numPts(); + tmp_G.mult( normalization ); + + // Copy from tmp_G to phi + phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect() ); + + // Loop to destroy FFT plans + for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + ablastr::math::anyfft::DestroyPlan(forward_plan_G[mfi]); + ablastr::math::anyfft::DestroyPlan(forward_plan_rho[mfi]); + ablastr::math::anyfft::DestroyPlan(backward_plan[mfi]); + } +} +} // namespace ablastr::fields diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package new file mode 100644 index 00000000000..4fb7bd88e5f --- /dev/null +++ b/Source/ablastr/fields/Make.package @@ -0,0 +1,7 @@ +ifeq ($(USE_PSATD),TRUE) + ifeq ($(DIM),3) + CEXE_sources += IntegratedGreenFunctionSolver.cpp + endif +endif + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr/fields diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 9c6053bed07..da0078f8b5a 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -11,8 +11,13 @@ #include #include #include +#include #include +#include +#if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) +#include +#endif #include #include @@ -77,6 +82,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler + * \param[in] is_solver_multigrid boolean to select the Poisson solver: 1 for Multigrid, 0 for FFT * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -96,10 +102,11 @@ computePhi (amrex::Vector const & rho, amrex::Real absolute_tolerance, int const max_iters, int const verbosity, - amrex::Vector const geom, - amrex::Vector const dmap, - amrex::Vector const grids, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, + [[maybe_unused]] bool is_solver_multigrid, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, @@ -109,6 +116,8 @@ computePhi (amrex::Vector const & rho, { using namespace amrex::literals; + ABLASTR_PROFILE("computePhi", false); + if (!rel_ref_ratio.has_value()) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, "rel_ref_ratio must be set if mesh-refinement is used"); @@ -117,11 +126,9 @@ computePhi (amrex::Vector const & rho, auto const finest_level = static_cast(rho.size() - 1); - // scale rho appropriately; also determine if rho is zero everywhere + // determine if rho is zero everywhere amrex::Real max_norm_b = 0.0; for (int lev=0; lev<=finest_level; lev++) { - using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! max_norm_b = amrex::max(max_norm_b, rho[lev]->norm0()); } amrex::ParallelDescriptor::ReduceRealMax(max_norm_b); @@ -153,6 +160,27 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif +#if (defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D)) + // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected + if(!is_solver_multigrid && lev==0){ + amrex::Array const dx_igf + {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), + geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), + geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; + if ( max_norm_b == 0 ) { + phi[lev]->setVal(0); + } else { + computePhiIGF( *rho[lev], *phi[lev], dx_igf, grids[lev] ); + } + continue; + } +#endif + + // Use the Multigrid (MLMG) solver if selected or on refined patches + // but first scale rho appropriately + using namespace ablastr::constant::SI; + rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + #if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) // Determine whether to use semi-coarsening amrex::Array dx_scaled diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 6e68d0fadb5..d49335723d8 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -89,9 +89,9 @@ computeVectorPotential ( amrex::Vector > co amrex::Real absolute_tolerance, int const max_iters, int const verbosity, - amrex::Vector const geom, - amrex::Vector const dmap, - amrex::Vector const grids, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, @@ -185,11 +185,10 @@ computeVectorPotential ( amrex::Vector > co relative_tolerance, absolute_tolerance ); // Synchronize the ghost cells, do halo exchange - ablastr::utils::communication::FillBoundary(*A[lev][adim], - A[lev][adim]->nGrowVect(), - WarpX::do_single_precision_comms, - geom[lev].periodicity(), - false); + ablastr::utils::communication::FillBoundary( + *A[lev][adim], A[lev][adim]->nGrowVect(), + do_single_precision_comms, + geom[lev].periodicity(), false); // needed for solving the levels by levels: // - coarser level is initial guess for finer level diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index 8ecea2a2486..b6b1706f807 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -8,6 +8,7 @@ #include "AnyFFT.H" #include "ablastr/utils/TextMsg.H" +#include "ablastr/profiler/ProfilerWrapper.H" namespace ablastr::math::anyfft { @@ -30,6 +31,7 @@ namespace ablastr::math::anyfft Complex * const complex_array, const direction dir, const int dim) { FFTplan fft_plan; + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan", false); // Initialize fft_plan.m_plan with the vendor fft plan. cufftResult result; @@ -69,10 +71,12 @@ namespace ablastr::math::anyfft void DestroyPlan(FFTplan& fft_plan) { + ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan", false); cufftDestroy( fft_plan.m_plan ); } void Execute(FFTplan& fft_plan){ + ABLASTR_PROFILE("ablastr::math::anyfft::Execute", false); // make sure that this is done on the same GPU stream as the above copy cudaStream_t stream = amrex::Gpu::Device::cudaStream(); cufftSetStream ( fft_plan.m_plan, stream); diff --git a/Source/ablastr/parallelization/MPIInitHelpers.H b/Source/ablastr/parallelization/MPIInitHelpers.H index 6b9bdfa4737..660e5e56d06 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.H +++ b/Source/ablastr/parallelization/MPIInitHelpers.H @@ -15,7 +15,7 @@ namespace ablastr::parallelization * * @return the MPI_THREAD_* level required for MPI_Init_thread */ - int + constexpr int mpi_thread_required (); /** Initialize MPI diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp index 633c004c93a..513035c483c 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.cpp +++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp @@ -29,20 +29,20 @@ namespace ablastr::parallelization { - int + constexpr int mpi_thread_required () { - int thread_required = -1; #ifdef AMREX_USE_MPI - thread_required = MPI_THREAD_SINGLE; // equiv. to MPI_Init -# ifdef AMREX_USE_OMP - thread_required = MPI_THREAD_FUNNELED; -# endif # ifdef AMREX_MPI_THREAD_MULTIPLE // i.e. for async_io - thread_required = MPI_THREAD_MULTIPLE; + return MPI_THREAD_MULTIPLE; +# elif AMREX_USE_OMP + return MPI_THREAD_FUNNELED; +# else + return MPI_THREAD_SINGLE; // equiv. to MPI_Init # endif +#else + return -1; #endif - return thread_required; } std::pair< int, int > @@ -58,7 +58,7 @@ namespace ablastr::parallelization } #endif - const int thread_required = mpi_thread_required(); + constexpr int thread_required = mpi_thread_required(); #ifdef AMREX_USE_MPI int thread_provided = -1; MPI_Init_thread(&argc, &argv, thread_required, &thread_provided); @@ -82,7 +82,7 @@ namespace ablastr::parallelization check_mpi_thread_level () { #ifdef AMREX_USE_MPI - const int thread_required = mpi_thread_required(); + constexpr int thread_required = mpi_thread_required(); int thread_provided = -1; MPI_Query_thread(&thread_provided); auto mtn = amrex::ParallelDescriptor::mpi_level_to_string; diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index f43e35c6b0b..b2dedccd03e 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -44,9 +44,6 @@ namespace ablastr::particles * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit - * \param cost pointer to (load balancing) cost corresponding to box where present - particles deposit current. If nullptr, costs are not updated. (default: nullptr) - * \param load_balance_costs_update_algo selected method for updating load balance costs (default: 0) * \param do_device_synchronize call amrex::Gpu::synchronize() for tiny profiler regions (default: true) */ template< typename T_PC > @@ -67,8 +64,6 @@ deposit_charge (typename T_PC::ParIterType& pti, long const offset = 0, std::optional np_to_deposit = std::nullopt, int const icomp = 0, int const nc = 1, - amrex::Real * const AMREX_RESTRICT cost = nullptr, - long const load_balance_costs_update_algo = 0, bool const do_device_synchronize = true) { // deposition guards @@ -76,7 +71,7 @@ deposit_charge (typename T_PC::ParIterType& pti, if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); } - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho.allLE(rho->nGrowVect()), "num_rho_deposition_guards are larger than allocated!"); // particle shape auto const[nox, noy, noz] = std::array{particle_shape, particle_shape, particle_shape}; @@ -183,23 +178,19 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 4){ doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); diff --git a/Source/ablastr/particles/NodalFieldGather.H b/Source/ablastr/particles/NodalFieldGather.H index 5a10c73ae20..f324264db7f 100644 --- a/Source/ablastr/particles/NodalFieldGather.H +++ b/Source/ablastr/particles/NodalFieldGather.H @@ -12,16 +12,19 @@ #include #include #include +#include #include #include + namespace ablastr::particles { + /** * \brief Compute weight of each surrounding node (or cell-centered nodes) in interpolating a nodal ((or a cell-centered node) field - * to the given coordinates. If nodal=1, then the calculations will be done with respect to the nodes (default). If nodal=0, then the calculations will be done with respect to the cell-centered nodal) - * - * This currently only does linear order. + * to the given coordinates. If template parameter IdxType is amrex::IndexType::CellIndex::NODE, then the calculations will be done with respect to the nodes. + * If template parameter IdxType is amrex::IndexType::CellIndex::CELL, then the calculations will be done with respect to the cell-centered nodal. + * This currently only does linear order. * * \param xp,yp,zp Particle position coordinates * \param plo Index lower bounds of domain. @@ -30,24 +33,30 @@ namespace ablastr::particles * \param W 2D array of weights to store each neighbouring node (or cell-centered node) * \param nodal Int that tells if the weights are calculated in respect to the nodes (nodal=1) of the cell-centered nodes (nodal=0) */ +template AMREX_GPU_HOST_DEVICE AMREX_INLINE void compute_weights (const amrex::ParticleReal xp, const amrex::ParticleReal yp, const amrex::ParticleReal zp, amrex::GpuArray const& plo, amrex::GpuArray const& dxi, - int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2], int nodal=1) noexcept + int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2]) noexcept { using namespace amrex::literals; -#if !((nodal==0)||(nodal==1)) - ABLASTR_ABORT_WITH_MESSAGE("Error: 'nodal' has to be equal to 0 or 1"); -#endif + constexpr auto half_if_cell_centered = [](){ + if constexpr (IdxType == amrex::IndexType::CellIndex::NODE){ + return 0.0_rt; + } + else{ + return 0.5_rt; + } + }(); #if (defined WARPX_DIM_3D) - const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; - const amrex::Real y = (yp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; - const amrex::Real z = (zp - plo[2]) * dxi[2] + static_cast(nodal-1)*0.5_rt; + const amrex::Real x = (xp - plo[0]) * dxi[0] - half_if_cell_centered; + const amrex::Real y = (yp - plo[1]) * dxi[1] - half_if_cell_centered; + const amrex::Real z = (zp - plo[2]) * dxi[2] - half_if_cell_centered; i = static_cast(amrex::Math::floor(x)); j = static_cast(amrex::Math::floor(y)); @@ -64,17 +73,17 @@ void compute_weights (const amrex::ParticleReal xp, #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) # if (defined WARPX_DIM_XZ) - const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real x = (xp - plo[0]) * dxi[0] - half_if_cell_centered; amrex::ignore_unused(yp); i = static_cast(amrex::Math::floor(x)); W[0][1] = x - i; # elif (defined WARPX_DIM_RZ) - const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] - half_if_cell_centered; i = static_cast(amrex::Math::floor(r)); W[0][1] = r - i; # endif - const amrex::Real z = (zp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; + const amrex::Real z = (zp - plo[1]) * dxi[1] - half_if_cell_centered; j = static_cast(amrex::Math::floor(z)); W[1][1] = z - j; @@ -83,8 +92,15 @@ void compute_weights (const amrex::ParticleReal xp, k = 0; #else - amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W, nodal); - ABLASTR_ABORT_WITH_MESSAGE("Error: compute_weights not yet implemented in 1D"); + const amrex::Real z = (zp - plo[0]) * dxi[0] - half_if_cell_centered; + amrex::ignore_unused(xp, yp); + i = static_cast(amrex::Math::floor(z)); + + W[0][1] = z - i; + W[0][0] = 1.0_rt - W[0][1]; + + j = 0; + k = 0; #endif } @@ -100,8 +116,8 @@ amrex::Real interp_field_nodal (int i, int j, int k, const amrex::Real W[AMREX_SPACEDIM][2], amrex::Array4 const& scalar_field) noexcept { -#if (defined WARPX_DIM_3D) amrex::Real value = 0; +#if (defined WARPX_DIM_3D) value += scalar_field(i, j , k ) * W[0][0] * W[1][0] * W[2][0]; value += scalar_field(i+1, j , k ) * W[0][1] * W[1][0] * W[2][0]; value += scalar_field(i, j+1, k ) * W[0][0] * W[1][1] * W[2][0]; @@ -111,15 +127,13 @@ amrex::Real interp_field_nodal (int i, int j, int k, value += scalar_field(i , j+1, k+1) * W[0][0] * W[1][1] * W[2][1]; value += scalar_field(i+1, j+1, k+1) * W[0][1] * W[1][1] * W[2][1]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Real value = 0; value += scalar_field(i, j , k) * W[0][0] * W[1][0]; value += scalar_field(i+1, j , k) * W[0][1] * W[1][0]; value += scalar_field(i, j+1, k) * W[0][0] * W[1][1]; value += scalar_field(i+1, j+1, k) * W[0][1] * W[1][1]; #else - const amrex::Real value = 0; - amrex::ignore_unused(i, j, k, W, scalar_field); - ABLASTR_ABORT_WITH_MESSAGE("Error: interp_field not yet implemented in 1D"); + value += scalar_field(i, j , k) * W[0][0]; + value += scalar_field(i+1, j , k) * W[0][1]; #endif return value; } @@ -144,7 +158,7 @@ amrex::Real doGatherScalarFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); return interp_field_nodal(ii, jj, kk, W, scalar_field); } @@ -172,7 +186,7 @@ doGatherVectorFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); amrex::GpuArray const field_interp = { interp_field_nodal(ii, jj, kk, W, vector_field_x), diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.H b/Source/ablastr/utils/msg_logger/MsgLogger.H index 01a61be5c80..401432f5dda 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.H +++ b/Source/ablastr/utils/msg_logger/MsgLogger.H @@ -196,7 +196,7 @@ namespace ablastr::utils::msg_logger * * @param[in] msg a Msg struct */ - void record_msg(Msg msg); + void record_msg(const Msg& msg); /** * \brief This function returns a vector containing the recorded messages diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index dde041652da..6537a8f61e5 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -216,7 +216,7 @@ Logger::Logger() : m_io_rank{amrex::ParallelDescriptor::IOProcessorNumber()} {} -void Logger::record_msg(Msg msg) +void Logger::record_msg(const Msg& msg) { m_messages[msg]++; } diff --git a/Source/ablastr/utils/text/StreamUtils.cpp b/Source/ablastr/utils/text/StreamUtils.cpp index 965beb179ba..541ef300f7c 100644 --- a/Source/ablastr/utils/text/StreamUtils.cpp +++ b/Source/ablastr/utils/text/StreamUtils.cpp @@ -7,7 +7,6 @@ #include "StreamUtils.H" -#include #include void diff --git a/Source/ablastr/warn_manager/WarnManager.H b/Source/ablastr/warn_manager/WarnManager.H index 737fb8fba73..56ff1bf1a1a 100644 --- a/Source/ablastr/warn_manager/WarnManager.H +++ b/Source/ablastr/warn_manager/WarnManager.H @@ -84,9 +84,9 @@ namespace ablastr::warn_manager * @param[in] priority priority of the warning message ("medium" by default) */ void RecordWarning( - std::string topic, - std::string text, - WarnPriority priority = WarnPriority::medium); + const std::string& topic, + const std::string& text, + const WarnPriority& priority = WarnPriority::medium); /** * \brief This function prints all the warning messages collected on the present MPI rank @@ -225,9 +225,9 @@ namespace ablastr::warn_manager * @param[in] priority priority of the warning message ("medium" by default) */ void WMRecordWarning( - std::string topic, - std::string text, - WarnPriority priority = WarnPriority::medium); + const std::string& topic, + const std::string& text, + const WarnPriority& priority = WarnPriority::medium); } #endif //ABLASTR_WARN_MANAGER_H_ diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index ee052ded299..b1a194b2633 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -55,9 +55,9 @@ WarnManager::WarnManager(): {} void WarnManager::RecordWarning( - std::string topic, - std::string text, - WarnPriority priority) + const std::string& topic, + const std::string& text, + const WarnPriority& priority) { auto msg_priority = abl_msg_logger::Priority::high; if(priority == WarnPriority::low) { @@ -316,9 +316,9 @@ WarnManager& ablastr::warn_manager::GetWMInstance() } void ablastr::warn_manager::WMRecordWarning( - std::string topic, - std::string text, - WarnPriority priority) + const std::string& topic, + const std::string& text, + const WarnPriority& priority) { WarnManager::GetInstance().RecordWarning( topic, text, priority); diff --git a/Tools/Algorithms/stencil.py b/Tools/Algorithms/stencil.py index 63bb7f11c2e..dde7398daaa 100644 --- a/Tools/Algorithms/stencil.py +++ b/Tools/Algorithms/stencil.py @@ -16,9 +16,9 @@ import sys sys.path.append('../Parser/') -from input_file_parser import parse_input_file import matplotlib.pyplot as plt import numpy as np +from input_file_parser import parse_input_file from scipy.constants import c plt.style.use('tableau-colorblind10') diff --git a/Tools/LibEnsemble/all_machine_specs.py b/Tools/LibEnsemble/all_machine_specs.py deleted file mode 100644 index 8dbc2879f61..00000000000 --- a/Tools/LibEnsemble/all_machine_specs.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It contains a dictionary for machine-specific elements. -""" - - -local_specs = { - 'name': 'local', # Machine name - 'cores': 1, # Number of cores per simulation - 'sim_app': os.environ['HOME'] + '/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex', - 'extra_args': '', # extra arguments passed to mpirun/mpiexec at execution - 'OMP_NUM_THREADS': '2', - 'sim_max': 3 # Maximum number of simulations -} - - -summit_specs = { - 'name': 'summit', # Machine name - 'sim_app': os.environ['HOME'] + '/warpx/Bin/main2d.gnu.TPROF.MPI.CUDA.ex', - # extra arguments passed to jsrun at execution - 'extra_args': '-n 1 -a 1 -g 1 -c 1 --bind=packed:1 --smpiargs="-gpu"', - 'OMP_NUM_THREADS': '1', - 'sim_max': 400 # Maximum number of simulations -} diff --git a/Tools/LibEnsemble/plot_results.py b/Tools/LibEnsemble/plot_results.py deleted file mode 100755 index 6c294fe3675..00000000000 --- a/Tools/LibEnsemble/plot_results.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python - -""" -Plotting script for LibEnsemble run on WarpX simulations. -input: A directory which which the script will read the -latest .npy and .pickle files generated by LibEnsemble -""" - -import glob -import os -import pickle -import sys - -import matplotlib.pyplot as plt -import numpy as np - -# 'aposmm' or 'random' -generator_type = 'random' - -print("Reading latest .npy and .pickle files in " + sys.argv[1]) - -# Read latest .npy file, and store the result in results -# * means all if need specific format then *.csv -list_of_files = glob.glob(sys.argv[1] + '*npy') -latest_file = max(list_of_files, key=os.path.getctime) -results = np.load(latest_file) - -# Read latest .pickle file, and store the result in pickle_data -# * means all if need specific format then *.csv -list_of_files = glob.glob(sys.argv[1] + '*pickle') -latest_file = max(list_of_files, key=os.path.getctime) -with open(latest_file, 'rb') as f: - pickle_data = pickle.load(f) -if generator_type == 'aposmm': - nbatches = len(pickle_data[1]['run_order']) - -# Remove un-returned (i.e., not submitted) simulations -results = results[results['returned'] == 1] - -# Re-organize results into a dictionary. Each key is a field in libE_output, -# each value is an array with the value of this field for all WarpX simulations -names = results.dtype.names -results_dict = {} -for i in range(len(names)): - results_dict[names[i]] = np.array([task[i] for task in results]) - -# Sanity checks: exit if a simulation has null charge, as the script -# doesn't handle this (shouldn't be too hard to add though). -if np.min(results_dict['charge'] == 0.0): - sys.exit() -if np.min(results_dict['f'] == 0.0): - sys.exit() - -# Custom: which output parameters are plotted -# as a function of which input parameters -plot_output = ['f', 'emittance', 'charge', 'energy_avg', 'energy_std'] -plot_input = ['ramp_down_1', 'zlens_1', 'adjust_factor', 'ramp_down_2'] - -# For convenience, dictionary with pyplot.ylim -# tuples for all elements in plot_output -d_ylim = { - 'f': (3.e-7, 5.e-3), - 'emittance': (3.e-7, 5.e-3), - 'charge': (1.e-7, 1.e-5), - 'energy_avg': (5.e1, 5.e2), - 'energy_std': (1.e-2, 1.e-1) -} - -# For convenience, dictionary with units for all elements in plot_output -d_yunits = { - 'f': " (a.u.)", - 'emittance': " (m rad)", - 'charge': " (pC/m)", - 'energy_avg': " (MeV)", - 'energy_std': " (%)" -} - -# Print input and output parameters for the optimal run -print("Best run:") -ind_best = np.nanargmin(results_dict['f']) -for key in plot_input + plot_output: - print(key, results_dict[key][ind_best]) -print("charge_f/charge_i ", - results[ind_best]['charge']/np.nanmax(results_dict['charge'])) - -# And now plotting begins -plt.figure(figsize=(16, 10)) -plt.style.use('dark_background') -# Loop over all input parameters in plot_input -for icount, iname in enumerate(plot_input): - # Loop over all output parameters in plot_output - for count, name in enumerate(plot_output): - plt.subplot(len(plot_input), len(plot_output), - len(plot_output)*icount + count+1) - if generator_type == 'aposmm': - # Plot 1 point per run. Runs that are part of the same local - # minimum search are batched together and plotted with the - # same color. - for batch in pickle_data[1]['run_order']: - my_sims = np.isin(results_dict['sim_id'], - pickle_data[1]['run_order'][batch]) - plt.scatter(np.squeeze(results_dict[iname][my_sims]), - np.squeeze(results_dict[name][my_sims]), - s=2) - elif generator_type == 'random': - # Plot 1 point per run. Colorbar stands for sim ID - # (larger value = later run). - plt.scatter(np.squeeze(results_dict[iname]), - np.squeeze(results_dict[name]), - c=results_dict['sim_id'], s=2, cmap='viridis') - cbar = plt.colorbar() - cbar.ax.set_ylabel('start time (arb. units)') - else: - print("generator_type should be either random or aposmm") - sys.exit() - plt.yscale('log') - plt.xlim(np.min(results_dict[iname]), np.max(results_dict[iname])) - plt.ylim(d_ylim[name]) - plt.grid() - plt.title(name) - plt.xlabel(iname) - plt.ylabel(name + d_yunits[name]) - -plt.tight_layout() - -# Save figure -plt.savefig('results.pdf', bbox_inches='tight') -plt.savefig('results.png', bbox_inches='tight') diff --git a/Tools/LibEnsemble/read_sim_output.py b/Tools/LibEnsemble/read_sim_output.py deleted file mode 100644 index 5eabf18030a..00000000000 --- a/Tools/LibEnsemble/read_sim_output.py +++ /dev/null @@ -1,115 +0,0 @@ -import glob -import os -import shutil - -import numpy as np -import scipy.constants as scc -import yt - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It reads the output plotfiles of a simulation and returns beam -quantity relevant for the LibEnsemble optimizations ('f') as well as other -beam quantities for convenience. -""" - -yt.funcs.mylog.setLevel(50) - -def slice_emittance(x, xp, g, b, w): - xpgb = xp * g * b - xav = np.average(x, weights = w) - xpav = np.average(xpgb, weights = w) - xstd2 = np.average((x - xav)**2, weights = w) - xpstd2 = np.average((xpgb - xpav)**2, weights = w) - xp2 = (np.average((x - xav) * (xpgb-xpav), weights = w))**2 - em = np.sqrt(xstd2 * xpstd2 - xp2) - return em - -def _beam_properties(filepath): - """ - Reads plotfile filepath and compute and return beam parameters - - Parameters - ---------- - filepath : path to plotfile to read - """ - - # Read beam quantities from plotfile - ds = yt.load(filepath) - - if ('beam' in [i[0] for i in ds.field_list]): - ad = ds.all_data() - w = ad['beam', 'particle_weight'].v - if (w.shape[0] <= 200): - print('Insufficient particles in ',filepath) - return 0.0, 0.0, 0.0, 0.0 - else: - x = ad['beam', 'particle_position_x'].v - z = ad['beam', 'particle_position_y'].v - ux = ad['beam', 'particle_momentum_x'].v/scc.m_e/scc.c - uy = ad['beam', 'particle_momentum_y'].v/scc.m_e/scc.c - uz = ad['beam', 'particle_momentum_z'].v/scc.m_e/scc.c - - # Compute beam parameters - # Defined like that, the beam charge is > 0. - charge = np.sum(w) * scc.e - gamma = np.sqrt(1. + ux**2 + uy**2 + uz**2) - beta = np.sqrt(1.0 - 1.0 / gamma**2) - energy_MeV = scc.physical_constants["electron mass energy equivalent in MeV"][0] * (gamma - 1.) - energy_avg = np.average(energy_MeV, weights = w) - energy_std = np.sqrt(np.average((energy_MeV - energy_avg)**2, weights = w)) / energy_avg * 100 - nslices = 20 - zslices = np.linspace(np.min(z), np.max(z), nslices+1) - exlist = np.zeros(nslices) - for slicei in range(nslices): - cond = [(z > zslices[slicei]) & (z < zslices[slicei+1])][0] - wslice = w[cond] - if (wslice.shape[0] > 10): - xslice = x[cond] - xpslice = ux[cond]/uz[cond] - gslice = gamma[cond] - bslice = beta[cond] - exlist[slicei] = slice_emittance(xslice, xpslice, gslice, bslice, wslice) - emittance = np.mean(exlist) - return charge, energy_avg, energy_std, emittance - else: - print('No beam particles in ',filepath) - return 0.0, 0.0, 0.0, 0.0 - -def read_sim_output(workdir): - """ - Return optimizing quantity 'f' and other parameters for convenience. - - Parameters - ---------- - workdir : Path to directory where the simulation ran. - """ - # Get beam properties at the beginning of the run - file_list = glob.glob('diags/plotfiles/plt?????') - if (len(file_list) <2): - print(workdir,' did not have final plotfile') - return np.array([np.nan, np.nan, np.nan, np.nan, np.nan]) - file_list.sort() - datafile = file_list[0] - filepath = os.path.join(workdir, datafile) - charge_i, _, _, emittance_i = _beam_properties(filepath) - - # Get beam properties at the end of the run - datafile = file_list[-1] - filepath = os.path.join(workdir, datafile) - charge_f, energy_avg, energy_std, emittance_f = _beam_properties(filepath) - - if (charge_f > 0.0): - # delete simulation results, just to have smaller data - shutil.rmtree('diags') - - # Build a quantity to minimize (f) that encompasses - # emittance AND charge loss 1% charge loss has the - # same impact as doubling the initial emittance. - # we minimize f! - f = emittance_f + emittance_i*(1.-charge_f/charge_i)*100 - warpx_out = np.array([f, energy_std, energy_avg, charge_f, emittance_f]) - - return warpx_out - else: - return np.array([np.nan, np.nan, np.nan, np.nan, np.nan]) diff --git a/Tools/LibEnsemble/requirements.txt b/Tools/LibEnsemble/requirements.txt deleted file mode 100644 index 88217255e85..00000000000 --- a/Tools/LibEnsemble/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -libensemble -matplotlib -nlopt -numpy -pytest -scipy -yt diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py deleted file mode 100644 index b86c0249d01..00000000000 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It is the entry point script that runs LibEnsemble. Libensemble -then launches WarpX simulations. - -Execute locally via the following command: - python run_libensemble_on_warpx.py --comms local --nworkers 3 -On summit, use the submission script: - bsub summit_submit_mproc.sh - -The number of concurrent evaluations of the objective function will be -nworkers=1 as one worker is for the persistent gen_f. -""" - -# Either 'random' or 'aposmm' -generator_type = 'aposmm' -# Either 'local' or 'summit' -machine = 'local' - -import sys - -# Import libEnsemble modules -from libensemble.libE import libE -import numpy as np -from warpx_simf import run_warpx # Sim function from current directory - -if generator_type == 'random': - from libensemble.alloc_funcs.give_sim_work_first import \ - give_sim_work_first as alloc_f - from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -elif generator_type == 'aposmm': - import libensemble.gen_funcs - libensemble.gen_funcs.rc.aposmm_optimizers = 'nlopt' - from libensemble.alloc_funcs.persistent_aposmm_alloc import \ - persistent_aposmm_alloc as alloc_f - from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f -else: - print("you shouldn' hit that") - sys.exit() - -import all_machine_specs -from libensemble import libE_logger -from libensemble.executors.mpi_executor import MPIExecutor -from libensemble.tools import ( - add_unique_random_streams, - parse_args, - save_libE_output, -) - -# Import machine-specific run parameters -if machine == 'local': - machine_specs = all_machine_specs.local_specs -elif machine == 'summit': - machine_specs = all_machine_specs.summit_specs -else: - print("you shouldn' hit that") - sys.exit() - -libE_logger.set_level('INFO') - -nworkers, is_master, libE_specs, _ = parse_args() - -# Set to full path of warp executable -sim_app = machine_specs['sim_app'] - -# Problem dimension. This is the number of input parameters exposed, -# that LibEnsemble will vary in order to minimize a single output parameter. -n = 4 - -exctr = MPIExecutor(central_mode=True) -exctr.register_calc(full_path=sim_app, calc_type='sim') - -# State the objective function, its arguments, output, and necessary parameters -# (and their sizes). Here, the 'user' field is for the user's (in this case, -# the simulation) convenience. Feel free to use it to pass number of nodes, -# number of ranks per note, time limit per simulation etc. -sim_specs = { - # Function whose output is being minimized. The parallel WarpX run is - # launched from run_WarpX. - 'sim_f': run_warpx, - # Name of input for sim_f, that LibEnsemble is allowed to modify. - # May be a 1D array. - 'in': ['x'], - 'out': [ - # f is the single float output that LibEnsemble minimizes. - ('f', float), - # All parameters below are not used for calculation, - # just output for convenience. - # Final relative energy spread. - ('energy_std', float, (1,)), - # Final average energy, in MeV. - ('energy_avg', float, (1,)), - # Final beam charge. - ('charge', float, (1,)), - # Final beam emittance. - ('emittance', float, (1,)), - # input parameter: length of first downramp. - ('ramp_down_1', float, (1,)), - # input parameter: Length of second downramp. - ('ramp_down_2', float, (1,)), - # input parameter: position of the focusing lens. - ('zlens_1', float, (1,)), - # Relative strength of the lens (1. is from - # back-of-the-envelope calculation) - ('adjust_factor', float, (1,)), - ], - 'user': { - # name of input file - 'input_filename': 'inputs', - # Run timeouts after 3 mins - 'sim_kill_minutes': 3, - # machine-specific parameters - 'machine_specs': machine_specs - } -} - -# State the generating function, its arguments, output, -# and necessary parameters. -if generator_type == 'random': - # Here, the 'user' field is for the user's (in this case, - # the RNG) convenience. - gen_specs = { - # Generator function. Will randomly generate new sim inputs 'x'. - 'gen_f': gen_f, - # Generator input. This is a RNG, no need for inputs. - 'in': [], - 'out': [ - # parameters to input into the simulation. - ('x', float, (n,)) - ], - 'user': { - # Total max number of sims running concurrently. - 'gen_batch_size': nworkers, - # Lower bound for the n parameters. - 'lb': np.array([2.e-3, 2.e-3, 0.005, .1]), - # Upper bound for the n parameters. - 'ub': np.array([2.e-2, 2.e-2, 0.028, 3.]), - } - } - - alloc_specs = { - # Allocator function, decides what a worker should do. - # We use a LibEnsemble allocator. - 'alloc_f': alloc_f, - 'out': [ - ('allocated', bool) - ], - 'user': { - # If true wait for all sims to process before generate more - 'batch_mode': True, - # Only one active generator at a time - 'num_active_gens': 1 - } - } - -elif generator_type == 'aposmm': - # Here, the 'user' field is for the user's (in this case, - # the optimizer) convenience. - gen_specs = { - # Generator function. Will randomly generate new sim inputs 'x'. - 'gen_f': gen_f, - 'in': [], - 'out': [ - # parameters to input into the simulation. - ('x', float, (n,)), - # x scaled to a unique cube. - ('x_on_cube', float, (n,)), - # unique ID of simulation. - ('sim_id', int), - # Whether this point is a local minimum. - ('local_min', bool), - # whether the point is from a local optimization run - # or a random sample point. - ('local_pt', bool) - ], - 'user': { - # Number of sims for initial random sampling. - # Optimizer starts afterwards. - 'initial_sample_size': max(nworkers-1, 1), - # APOSMM/NLOPT optimization method - 'localopt_method': 'LN_BOBYQA', - 'num_pts_first_pass': nworkers, - # Relative tolerance of inputs - 'xtol_rel': 1e-3, - # Absolute tolerance of output 'f'. Determines when - # local optimization stops. - 'ftol_abs': 3e-8, - # Lower bound for the n input parameters. - 'lb': np.array([2.e-3, 2.e-3, 0.005, .1]), - # Upper bound for the n input parameters. - 'ub': np.array([2.e-2, 2.e-2, 0.028, 3.]), - } - } - - alloc_specs = { - # Allocator function, decides what a worker should do. - # We use a LibEnsemble allocator. - 'alloc_f': alloc_f, - 'out': [('given_back', bool)], - 'user': {}} - -else: - print("you shouldn' hit that") - sys.exit() - -# Save H to file every N simulation evaluations -libE_specs['save_every_k_sims'] = 100 -# Sim directory to be copied for each worker -libE_specs['sim_input_dir'] = 'sim' - -sim_max = machine_specs['sim_max'] # Maximum number of simulations -exit_criteria = {'sim_max': sim_max} # Exit after running sim_max simulations - -# Create a different random number stream for each worker and the manager -persis_info = add_unique_random_streams({}, nworkers + 1) - -# Run LibEnsemble, and store results in history array H -H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs) - -# Save results to numpy file -if is_master: - save_libE_output(H, persis_info, __file__, nworkers) diff --git a/Tools/LibEnsemble/sim/inputs b/Tools/LibEnsemble/sim/inputs deleted file mode 100644 index 03e9968e5b0..00000000000 --- a/Tools/LibEnsemble/sim/inputs +++ /dev/null @@ -1,215 +0,0 @@ -################################# -########## MESH PATCH ########### -################################# -amr.max_level = 0 - -################################# -######### BOX PARAMETERS ######## -################################# -# warpx.zmax_plasma_to_compute_max_step = 0.33 -warpx.zmax_plasma_to_compute_max_step = -1. -# max_step = 3150 - -amr.n_cell = 32 2048 # 64 3072 - -amr.max_grid_size = 4096 -amr.blocking_factor = 16 - -geometry.dims = 2 -geometry.is_periodic = 0 0 -# physical domain when running in the lab frame -geometry.prob_lo = -0.00024190484157981564 -0.00016126989438654374 -geometry.prob_hi = 0.00024190484157981564 0.0 - -################################# -############ NUMERICS ########### -################################# -warpx.verbose = 1 -algo.current_deposition = esirkepov -algo.charge_deposition = standard -#algo.field_gathering = standard -algo.field_gathering = momentum-conserving -algo.particle_pusher = vay -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 -algo.particle_shape = 3 -warpx.use_filter = 1 -warpx.filter_npass_each_dir = 1 4 -warpx.cfl = .9999 -warpx.do_pml = 0 -warpx.do_dynamic_scheduling = 1 -# Moving window -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # in units of the speed of light - -################################# -####### BOOST PARAMETERS ######## -################################# -warpx.gamma_boost = 30.0 -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = electrons ions electrons2 ions2 beam - -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = beam - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 1 1 -electrons.momentum_distribution_type = "at_rest" -electrons.xmin = -150.e-6 -electrons.xmax = 150.e-6 -electrons.ymin = -150.e-6 -electrons.ymax = 150.e-6 -electrons.zmin = 0.0 -electrons.zmax = 0.32 -electrons.profile = "predefined" -electrons.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = NUniformPerCell -ions.num_particles_per_cell_each_dim = 1 1 -ions.momentum_distribution_type = "at_rest" -ions.xmin = -150.e-6 -ions.xmax = 150.e-6 -ions.ymin = -150.e-6 -ions.ymax = 150.e-6 -ions.zmin = 0.0 -ions.zmax = 0.32 -ions.profile = "predefined" -ions.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 -ions.do_continuous_injection = 1 - -electrons2.charge = -q_e -electrons2.mass = m_e -electrons2.injection_style = NUniformPerCell -electrons2.num_particles_per_cell_each_dim = 1 1 1 -electrons2.momentum_distribution_type = "at_rest" -electrons2.xmin = -150.e-6 -electrons2.xmax = 150.e-6 -electrons2.ymin = -150.e-6 -electrons2.ymax = 150.e-6 -electrons2.zmin = 0.3485 -electrons2.zmax = 0.6685 -electrons2.profile = "predefined" -electrons2.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons2.predefined_profile_params = 0.3485 .02 .297 .003 40.e-6 1.7e23 -electrons2.do_continuous_injection = 1 - -ions2.charge = q_e -ions2.mass = m_p -ions2.injection_style = NUniformPerCell -ions2.num_particles_per_cell_each_dim = 1 1 1 -ions2.momentum_distribution_type = "at_rest" -ions2.xmin = -150.e-6 -ions2.xmax = 150.e-6 -ions2.ymin = -150.e-6 -ions2.ymax = 150.e-6 -ions2.zmin = 0.3485 -ions2.zmax = 0.6685 -ions2.profile = "predefined" -ions2.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions2.predefined_profile_params = 0.3485 .02 .297 .003 40.e-6 1.7e23 -ions2.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = 6.e-7 -beam.y_rms = 6.e-7 -beam.z_rms = 3.e-6 -beam.x_cut = 3. -beam.y_cut = 3. -beam.z_cut = 2. -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -98.e-6 -beam.npart = 100000 -beam.q_tot = -0.9e-12 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0. -beam.uy_m = 0. -beam.uz_m = 1956.9469069265976 -beam.ux_th = 0.4166666666666667 -beam.uy_th = 0. -beam.uz_th = 39.138943248532286 -beam.zinject_plane = 0.02 -beam.rigid_advance = true - -################################# -######### Lens Mirror ########### -################################# - -my_constants.ga = 13330 -my_constants.zlen = 0.3375 -my_constants.dlen = 0.019 -my_constants.wlen = 0.002 -my_constants.mcce = 510999. -my_constants.gab = 30.0 -my_constants.vb = 299625860.344456 -my_constants.adjust_factor = 1. - -particles.E_ext_particle_init_style = parse_E_ext_particle_function -particles.Ex_external_particle_function(x,y,z,t) = "( adjust_factor * gab * 2 * mcce * ga * x / (wlen * dlen) ) * ((gab*(z+vb*t))>=zlen) * ((gab*(z+vb*t))<=(zlen+wlen))" -particles.Ey_external_particle_function(x,y,z,t) = "0." -particles.Ez_external_particle_function(x,y,z,t) = "0." - -particles.B_ext_particle_init_style = parse_B_ext_particle_function -particles.Bx_external_particle_function(x,y,z,t) = "0." -particles.By_external_particle_function(x,y,z,t) = "( - adjust_factor * vb * gab * 2 * mcce * ga * x / (wlen * dlen * clight * clight) ) * ((gab*(z+vb*t))>=zlen) * ((gab*(z+vb*t))<=(zlen+wlen))" -particles.Bz_external_particle_function(x,y,z,t) = "0." - -warpx.num_mirrors=2 -warpx.mirror_z = 0.321 0.6695 -warpx.mirror_z_width = 8.0e-6 8.0e-6 -warpx.mirror_z_npoints = 4 4 - -################################# -############# LASER ############# -################################# -lasers.names = laser1 laser2 - -laser1.profile = Gaussian -laser1.position = 0. 0. -1.e-9 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 0. 1. 0. # The main polarization vector -laser1.e_max = 6.82274e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 50.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 7.33841e-14 # The duration of the laser (in seconds) -laser1.profile_t_peak = 1.46764864e-13 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 0.00875 # Focal distance from the antenna (in meters) -laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) - -laser2.profile = Gaussian -laser2.position = 0. 0. 0.348499999 # This point is on the laser plane -laser2.direction = 0. 0. 1. # The plane normal direction -laser2.polarization = 0. 1. 0. # The main polarization vector -laser2.e_max = 6.82274e12 # Maximum amplitude of the laser field (in V/m) -laser2.profile_waist = 50.e-6 # The waist of the laser (in meters) -laser2.profile_duration = 7.33841e-14 # The duration of the laser (in seconds) -laser2.profile_t_peak = 1.1626176366295598e-09 # The time at which the laser reaches its peak (in seconds) -laser2.profile_focal_distance = 0.00875 # Focal distance from the antenna (in meters) -laser2.wavelength = 0.8e-6 # The wavelength of the laser (in meters) -laser2.do_continuous_injection = 1 # Lasers are initialized outside of initial box - -diagnostics.diags_names = diag -diag.file_prefix = diags/plotfiles/plt -diag.diag_type = Full -diag.intervals = 10000 -diag.species = electrons electrons2 beam -diag.electrons.variables = w -diag.electrons2.variables = w -diag.beam.variables = w ux uy uz diff --git a/Tools/LibEnsemble/summit_submit_mproc.sh b/Tools/LibEnsemble/summit_submit_mproc.sh deleted file mode 100755 index d83c57b0e1e..00000000000 --- a/Tools/LibEnsemble/summit_submit_mproc.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -x -#BSUB -P -#BSUB -J libe_mproc -#BSUB -W 00:30 -#BSUB -nnodes 4 -#BSUB -alloc_flags "smt1" - -# Script to run libEnsemble using multiprocessing on launch nodes. -# Assumes Conda environment is set up. - -# To be run with central job management -# - Manager and workers run on launch node. -# - Workers submit tasks to the nodes in the job available. - -# Name of calling script- -export EXE=run_libensemble_on_warpx.py - -# Communication Method -export COMMS="--comms local" - -# Number of workers. -export NWORKERS="--nworkers 24" - -# Wallclock for libE. Slightly smaller than job wallclock -#export LIBE_WALLCLOCK=15 # Optional if pass to script - -# Name of Conda environment -export CONDA_ENV_NAME=libensemble - -export LIBE_PLOTS=true # Require plot scripts in $PLOT_DIR (see at end) -export PLOT_DIR=.. - -# Need these if not already loaded - -# module load python -# module load gcc -# module load cuda - -module unload xalt - -# Activate conda environment -export PYTHONNOUSERSITE=1 -. activate $CONDA_ENV_NAME - -# hash -d python # Check pick up python in conda env -hash -r # Check no commands hashed (pip/python...) - -# Launch libE. -#python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 -python $EXE $COMMS $NWORKERS > out.txt 2>&1 - -if [[ $LIBE_PLOTS = "true" ]]; then - python $PLOT_DIR/plot_libe_calcs_util_v_time.py - python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libe_histogram.py -fi diff --git a/Tools/LibEnsemble/warpx_simf.py b/Tools/LibEnsemble/warpx_simf.py deleted file mode 100644 index 4a1f0f90dee..00000000000 --- a/Tools/LibEnsemble/warpx_simf.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import time - -from libensemble.executors.executor import Executor -from libensemble.message_numbers import TASK_FAILED, WORKER_DONE -import numpy as np -from read_sim_output import read_sim_output -from write_sim_input import write_sim_input - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It defines a sim_f function that takes LibEnsemble history and -input parameters, run a WarpX simulation and returns 'f'. -""" - - -def run_warpx(H, persis_info, sim_specs, libE_info): - """ - This function runs a WarpX simulation and returns quantity 'f' as well as - other physical quantities measured in the run for convenience. Status check - is done periodically on the simulation, provided by LibEnsemble. - """ - - # Setting up variables needed for input and output - # keys = variable names - # x = variable values - # libE_output = what will be returned to libE - - calc_status = 0 # Returns to worker - - input_file = sim_specs['user']['input_filename'] - time_limit = sim_specs['user']['sim_kill_minutes'] * 60.0 - machine_specs = sim_specs['user']['machine_specs'] - - exctr = Executor.executor # Get Executor - - # Modify WarpX input file with input parameters calculated by gen_f - # and passed to this sim_f. - write_sim_input(input_file, H['x']) - - # Passed to command line in addition to the executable. - # Here, only input file - app_args = input_file - os.environ["OMP_NUM_THREADS"] = machine_specs['OMP_NUM_THREADS'] - - # Launch the executor to actually run the WarpX simulation - if machine_specs['name'] == 'summit': - task = exctr.submit(calc_type='sim', - extra_args=machine_specs['extra_args'], - app_args=app_args, - stdout='out.txt', - stderr='err.txt', - wait_on_run=True) - else: - task = exctr.submit(calc_type='sim', - num_procs=machine_specs['cores'], - app_args=app_args, - stdout='out.txt', - stderr='err.txt', - wait_on_run=True) - - # Periodically check the status of the simulation - poll_interval = 1 # secs - while(not task.finished): - time.sleep(poll_interval) - task.poll() - if task.runtime > time_limit: - task.kill() # Timeout - - # Set calc_status with optional prints. - if task.finished: - if task.state == 'FINISHED': - calc_status = WORKER_DONE - elif task.state == 'FAILED': - print("Warning: Task {} failed: Error code {}" - .format(task.name, task.errcode)) - calc_status = TASK_FAILED - elif task.state == 'USER_KILLED': - print("Warning: Task {} has been killed" - .format(task.name)) - else: - print("Warning: Task {} in unknown state {}. Error code {}" - .format(task.name, task.state, task.errcode)) - - # Safety - time.sleep(0.2) - - # Get output from a run and delete output files - warpx_out = read_sim_output(task.workdir) - - # Excluding results - NaN - from runs where beam was lost - if (warpx_out[0] != warpx_out[0]): - print(task.workdir, ' output led to NaN values (beam was lost or run did not finish)') - - # Pass the sim output values to LibEnsemble. - # When optimization is ON, 'f' is then passed to the generating function - # gen_f to generate new inputs for next runs. - # All other parameters are here just for convenience. - libE_output = np.zeros(1, dtype=sim_specs['out']) - libE_output['f'] = warpx_out[0] - libE_output['energy_std'] = warpx_out[1] - libE_output['energy_avg'] = warpx_out[2] - libE_output['charge'] = warpx_out[3] - libE_output['emittance'] = warpx_out[4] - libE_output['ramp_down_1'] = H['x'][0][0] - libE_output['ramp_down_2'] = H['x'][0][1] - libE_output['zlens_1'] = H['x'][0][2] - libE_output['adjust_factor'] = H['x'][0][3] - - return libE_output, persis_info, calc_status diff --git a/Tools/LibEnsemble/write_sim_input.py b/Tools/LibEnsemble/write_sim_input.py deleted file mode 100644 index e4a6398689e..00000000000 --- a/Tools/LibEnsemble/write_sim_input.py +++ /dev/null @@ -1,118 +0,0 @@ -import re - -import scipy.constants as scc - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It provides functions to modify some parameters in a WarpX -input file. -""" - - -def _set_value(str_text, str_line, val): - """ - This function search for str_line in str_text and overwrites the whole - matching line with str(val) - """ - str_text = re.sub('\n' + str_line + '.*', - '\n' + str_line + str(val), - str_text) - return str_text - - -def write_sim_input(input_file, x_values): - """ - This function modifies input_file to replace the values of some parameters - by values provided in x_values. - - Parameters - ---------- - input_file : WarpX input file. - x_values : Specific value of input x for this run. - """ - - # Parameters exposed to optimization - ramp_down_1 = x_values[0][0] - ramp_down_2 = x_values[0][1] - zlens_1 = x_values[0][2] - adjust_factor = x_values[0][3] - - # Fixed parameters - ramp_up_1 = 0.02 - plateau_1 = 0.297 - ramp_up_2 = ramp_up_1 - plateau_2 = plateau_1 - gap_12 = .0285 - - end_stage_1 = ramp_up_1 + plateau_1 + ramp_down_1 - beg_stage_2 = end_stage_1 + gap_12 - end_stage_2 = beg_stage_2 + ramp_up_2 + plateau_2 + ramp_down_2 - - # Updating mirrors position - mirror_z_1 = end_stage_1 + 1.0e-3 - mirror_z_2 = end_stage_2 + 1.0e-3 - - # Updating laser 2 configuration - z_antenna_2 = beg_stage_2 - 1.0e-9 #with value used for stage 1 - z_t_peak_2 = z_antenna_2 / scc.c + 2 * 7.33841e-14 # using pulse duration - - # End simulation when beam has just escaped the last stage - gamma_b = 30. - zmax_stop_run = end_stage_2 - 55.e-6 * gamma_b**2 * 2. - - with open(input_file) as file_handler: - output_text = file_handler.read() - - # Set end of stage 1 - output_text = _set_value( - output_text, 'electrons.zmax = ', str(end_stage_1)) - output_text = _set_value( - output_text, 'ions.zmax = ', str(end_stage_1)) - # Set length of final downramp of stage 1 - output_text = _set_value( - output_text, 'electrons.predefined_profile_params = ', - '0.0 .02 .297 ' + str(ramp_down_1) + ' 40.e-6 1.7e23') - output_text = _set_value( - output_text, 'ions.predefined_profile_params = ', - '0.0 .02 .297 ' + str(ramp_down_1) + ' 40.e-6 1.7e23') - # Set position of lens - output_text = _set_value( - output_text, 'my_constants.zlen = ', str(end_stage_1 + zlens_1)) - # Set beginning of stage 2 - output_text = _set_value( - output_text, 'electrons2.zmin = ', str(beg_stage_2)) - output_text = _set_value( - output_text, 'ions2.zmin = ', str(beg_stage_2)) - # Set end of stage 2 - output_text = _set_value( - output_text, 'electrons2.zmax = ', str(end_stage_2)) - output_text = _set_value( - output_text, 'ions2.zmax = ', str(end_stage_2)) - # Set length of final downramp of stage 2 - output_text = _set_value( - output_text, 'electrons2.predefined_profile_params = ', - str(beg_stage_2) + ' .02 .297 ' + str(ramp_down_2) + ' 40.e-6 1.7e23') - output_text = _set_value( - output_text, 'ions2.predefined_profile_params = ', - str(beg_stage_2) + ' .02 .297 ' + str(ramp_down_2) + ' 40.e-6 1.7e23') - # Set adjustment factor on lens strength - output_text = _set_value( - output_text, 'my_constants.adjust_factor = ', str(adjust_factor)) - # Set when to stop the run - output_text = _set_value( - output_text, 'warpx.zmax_plasma_to_compute_max_step = ', - str(zmax_stop_run)) - # Set mirrors position - output_text = _set_value( - output_text, 'warpx.mirror_z = ', str(mirror_z_1)+' '+str(mirror_z_2)) - # Set laser2 position - output_text = _set_value( - output_text, 'laser2.position = ', '0. 0. '+ str(z_antenna_2)) - # Set laser2 time until peak field is reached - output_text = _set_value( - output_text, 'laser2.profile_t_peak = ', str(z_t_peak_2)) - -# Write new input file - fout = open(input_file, 'w') - fout.write(output_text) - fout.close() diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh new file mode 100755 index 00000000000..39a96cc97d4 --- /dev/null +++ b/Tools/Linter/runClangTidy.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# Copyright 2024 Luca Fedeli +# +# This file is part of WarpX. +# + +# This script is a developer's tool to perform the +# checks done by the clang-tidy CI test locally. +# +# Note: this script is only tested on Linux + +echo "=============================================" +echo +echo "This script is a developer's tool to perform the" +echo "checks done by the clang-tidy CI test locally" +echo "_____________________________________________" + +# Check source dir +REPO_DIR=$(cd $(dirname ${BASH_SOURCE})/../../ && pwd) +echo +echo "Your current source directory is: ${REPO_DIR}" +echo "_____________________________________________" + +# Set number of jobs to use for compilation +PARALLEL="${WARPX_TOOLS_LINTER_PARALLEL:-4}" +echo +echo "${PARALLEL} jobs will be used for compilation." +echo "This can be overridden by setting the environment" +echo "variable WARPX_TOOLS_LINTER_PARALLEL, e.g.: " +echo +echo "$ export WARPX_TOOLS_LINTER_PARALLEL=8" +echo "$ ./Tools/Linter/runClangTidy.sh" +echo "_____________________________________________" + +# Check clang version +export CC="${CLANG:-"clang"}" +export CXX="${CLANGXX:-"clang++"}" +export CTIDY="${CLANGTIDY:-"clang-tidy"}" +echo +echo "The following versions of the clang compiler and" +echo "of the clang-tidy linter will be used:" +echo +echo "clang version:" +which ${CC} +${CC} --version +echo +echo "clang++ version:" +which ${CXX} +${CXX} --version +echo +echo "clang-tidy version:" +which ${CTIDY} +${CTIDY} --version +echo +echo "This can be overridden by setting the environment" +echo "variables CLANG, CLANGXX, and CLANGTIDY e.g.: " +echo "$ export CLANG=clang-15" +echo "$ export CLANGXX=clang++-15" +echo "$ export CTIDCLANGTIDYY=clang-tidy-15" +echo "$ ./Tools/Linter/runClangTidy.sh" +echo +echo "******************************************************" +echo "* Warning: clang v15 is currently used in CI tests. *" +echo "* It is therefore recommended to use this version. *" +echo "* Otherwise, a newer version may find issues not *" +echo "* currently covered by CI tests while older versions *" +echo "* may not find all the issues. *" +echo "******************************************************" +echo "_____________________________________________" + +# Prepare clang-tidy wrapper +echo +echo "Prepare clang-tidy wrapper" +echo "The following wrapper ensures that only source files" +echo "in WarpX/Source/* are actually processed by clang-tidy" +echo +cat > ${REPO_DIR}/clang_tidy_wrapper << EOF +#!/bin/bash +REGEX="[a-z_A-Z0-9\/]*WarpX\/Source[a-z_A-Z0-9\/]+.cpp" +if [[ \$4 =~ \$REGEX ]];then + ${CTIDY} \$@ +fi +EOF +chmod +x ${REPO_DIR}/clang_tidy_wrapper +echo "clang_tidy_wrapper: " +cat ${REPO_DIR}/clang_tidy_wrapper +echo "_____________________________________________" + +# Compile Warpx using clang-tidy +echo +echo "*******************************************" +echo "* Compile Warpx using clang-tidy *" +echo "* Please ensure that all the dependencies *" +echo "* required to compile WarpX are met *" +echo "*******************************************" +echo + +rm -rf ${REPO_DIR}/build_clang_tidy + +cmake -S ${REPO_DIR} -B ${REPO_DIR}/build_clang_tidy \ + -DCMAKE_CXX_CLANG_TIDY="${REPO_DIR}/clang_tidy_wrapper;--system-headers=0;--config-file=${REPO_DIR}/.clang-tidy" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DWarpX_DIMS="1;2;3;RZ" \ + -DWarpX_MPI=ON \ + -DWarpX_COMPUTE=OMP \ + -DWarpX_PSATD=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE + +cmake --build ${REPO_DIR}/build_clang_tidy -j ${PARALLEL} 2> ${REPO_DIR}/build_clang_tidy/clang-tidy.log + +cat ${REPO_DIR}/build_clang_tidy/clang-tidy.log +echo +echo "=============================================" diff --git a/Tools/PerformanceTests/run_automated.py b/Tools/PerformanceTests/run_automated.py index 73ab79b00df..f03ead05376 100644 --- a/Tools/PerformanceTests/run_automated.py +++ b/Tools/PerformanceTests/run_automated.py @@ -13,14 +13,14 @@ import sys import time +import git +import pandas as pd from functions_perftest import ( extract_dataframe, get_file_content, run_batch_nnode, store_git_hash, ) -import git -import pandas as pd # Get name of supercomputer and import configuration functions from # machine-specific file diff --git a/Tools/PerformanceTests/summit.py b/Tools/PerformanceTests/summit.py index cb0200e197c..c2ba6c70a2e 100644 --- a/Tools/PerformanceTests/summit.py +++ b/Tools/PerformanceTests/summit.py @@ -15,7 +15,7 @@ def executable_name(compiler,architecture): - return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.GPUCLOCK.ex' + return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.ex' def get_config_command(compiler, architecture): config_command = '' diff --git a/Tools/PostProcessing/plot_distribution_mapping.py b/Tools/PostProcessing/plot_distribution_mapping.py index 19628551f26..4b0cdfd532b 100644 --- a/Tools/PostProcessing/plot_distribution_mapping.py +++ b/Tools/PostProcessing/plot_distribution_mapping.py @@ -1,7 +1,7 @@ # Standard imports +import os from collections import defaultdict from itertools import product -import os # High-performance math import numpy as np diff --git a/Tools/PostProcessing/yt3d_mpi.py b/Tools/PostProcessing/yt3d_mpi.py index 20054756d22..655327aff3d 100644 --- a/Tools/PostProcessing/yt3d_mpi.py +++ b/Tools/PostProcessing/yt3d_mpi.py @@ -19,10 +19,10 @@ import glob -from mpi4py import MPI import numpy as np import scipy.constants as scc import yt +from mpi4py import MPI yt.funcs.mylog.setLevel(50) diff --git a/Tools/QedTablesUtils/CMakeLists.txt b/Tools/QedTablesUtils/CMakeLists.txt new file mode 100644 index 00000000000..09399316096 --- /dev/null +++ b/Tools/QedTablesUtils/CMakeLists.txt @@ -0,0 +1,35 @@ +# Common argument parser functions ############################################ +# +# TODO: Move to ABLASTR +set(arg_parser_src + Source/ArgParser/QedTablesArgParser.cpp + Source/ArgParser/QedTablesArgParser.H +) + + +# Build QED lookup tables generator ########################################### +# +add_executable(qed_table_generator + Source/QedTableGenerator.cpp + ${arg_parser_src} +) +add_executable(WarpX::qed_table_generator ALIAS qed_table_generator) + +target_link_libraries(qed_table_generator PRIVATE PXRMP_QED) + +target_compile_features(qed_table_generator PUBLIC cxx_std_17) +set_target_properties(qed_table_generator PROPERTIES CXX_EXTENSIONS OFF) + + +# Build QED lookup tables reader ############################################## +# +add_executable(qed_table_reader + Source/QedTableReader.cpp + ${arg_parser_src} +) +add_executable(WarpX::qed_table_reader ALIAS qed_table_reader) + +target_link_libraries(qed_table_reader PRIVATE PXRMP_QED) + +target_compile_features(qed_table_reader PUBLIC cxx_std_17) +set_target_properties(qed_table_reader PROPERTIES CXX_EXTENSIONS OFF) diff --git a/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H new file mode 100644 index 00000000000..f05b84ddca8 --- /dev/null +++ b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.H @@ -0,0 +1,58 @@ +#ifndef QED_TABLES_ARG_PARSER_ +#define QED_TABLES_ARG_PARSER_ + +#include +#include +#include +#include +#include +#include +#include + +namespace ArgParser +{ + /* Possible types of parameters of command line arguments (no argument, int, double, or std::string) */ + enum class ArgType { NoArg, Integer, Double, String }; + + /* This type represents the type of a command line argument: if it exists it is either an int, a double, or a std::string */ + using ArgVal = std::optional>; + + /* The type of a possible command line argument */ + using Key = std::tuple; + + /* The type of the map of the parsed command line arguments */ + using ParsedArgs = std::unordered_map; + + /** + * \brief Gets the value out of an ArgVal (std::optional>) object + * + * \tparam T the type to return (must be int, double, or std::string) + * \param[in] arg_val the ArgVal object + * \return the value in arg_val + */ + template + T GetVal(const ArgVal& arg_val) + { + return std::get(*arg_val); + } + + /** + * \brief Function to parse the command line arguments + * + * \param[in] keys the list of possible command line arguments + * \param[in] argc the number of command line arguments + * \param[in] argv all the command line arguments + * \return the parsed command line arguments + */ + ParsedArgs ParseArgs (const std::vector& keys, const int argc, char const* const* argv); + + /** + * \brief Prints the command line options + * + * \param[in] cmd_list the list of possible command line arguments + */ + void PrintHelp (const std::vector& cmd_list); +}; + + +#endif //QED_TABLES_ARG_PARSER_ diff --git a/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp new file mode 100644 index 00000000000..41d27477dcf --- /dev/null +++ b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp @@ -0,0 +1,108 @@ +#include "QedTablesArgParser.H" + +#include + +using namespace ArgParser; +using namespace std; + +namespace +{ +void +WarnMsg(const string& msg) +{ + cout << "!!! Parse Warning: " + msg + "\n"; +} + +void +ErrMsg(const string& msg) +{ + cout << "### Parse Error: " + msg + "\n"; + exit(1); +} + +vector +ArgvToStrVec(const int argc, char const* const* argv ) +{ + auto sargs = vector{}; + for (int i = 0; i < argc; ++i) + sargs.push_back(std::string{argv[i]}); + return sargs; +} + +optional> +GetArgType(const std::vector& keys, vector::iterator& it) +{ + const auto str = *it; + it++; + for (const auto& kk : keys){ + if (get<0>(kk) == str) + { + return std::make_pair(get<0>(kk), get<1>(kk)); + } + } + return nullopt; +} + +ArgVal +ReadArg(const ArgType arg_type, vector::iterator& it) +{ + if(arg_type == ArgType::NoArg) return nullopt; + if(arg_type == ArgType::String) return *(it++); + if(arg_type == ArgType::Integer) return stoi(*(it++)); + if(arg_type == ArgType::Double) return stod(*(it++)); + + ErrMsg("Failed to parse type!"s); + + return nullopt; +} + +} + +ParsedArgs +ArgParser::ParseArgs (const std::vector& keys, const int argc, char const* const* argv) +{ + auto parsed_args = ParsedArgs{}; + + auto sargs = ArgvToStrVec(argc, argv); + auto it = sargs.begin() + 1; // the first argument is always the executable name + while (it < sargs.end()){ + const auto tt = *it; + auto res = GetArgType(keys, it); + if (res != nullopt){ + const auto& [key, arg_type] = *res; + const auto arg = ReadArg(arg_type, it); + if (parsed_args.find(key) != parsed_args.end()) + WarnMsg("Rewriting '"+ key +"' argument !"); + parsed_args[key] = arg; + }else{ + WarnMsg("Can't parse '"+ tt +"' !"); + } + } + + return parsed_args; +} + +void +ArgParser::PrintHelp (const vector& cmd_list) +{ + cout << "Command line options: " << endl; + + for (const auto& el : cmd_list){ + const auto type = get<1>(el); + string stype = "[??????]"; + if (type == ArgType::NoArg) + stype = "[NO ARG]"; + else if (type == ArgType::String) + stype = "[STRING]"; + else if (type == ArgType::Double) + stype = "[DOUBLE]"; + else if (type == ArgType::Integer) + stype = "[INTEGR]"; + + cout << get<0>(el) + << " " << stype + << " " << get<2>(el) + << endl; + } + +} diff --git a/Tools/QedTablesUtils/Source/QedTableCommons.H b/Tools/QedTablesUtils/Source/QedTableCommons.H new file mode 100644 index 00000000000..40551b9e13c --- /dev/null +++ b/Tools/QedTablesUtils/Source/QedTableCommons.H @@ -0,0 +1,26 @@ +#ifndef QED_TABLES_COMMONS_ +#define QED_TABLES_COMMONS_ + +#include +#include + +template +bool Contains (const ContainerType& container, const ElementType& el) +{ + return container.find(el) != std::end(container); +} + +void AbortWithMessage(const std::string& msg) +{ + std::cout << "### ABORT : " << msg << std::endl; + std::cout << "___________________________" << std::endl; + exit(1); +} + +void SuccessExit() +{ + std::cout << "___________________________" << std::endl; + exit(0); +} + +#endif //QED_TABLES_COMMONS_ diff --git a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp new file mode 100644 index 00000000000..dfbde1df050 --- /dev/null +++ b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp @@ -0,0 +1,239 @@ +#include "QedTableCommons.H" +#include "ArgParser/QedTablesArgParser.H" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace ArgParser; +using namespace std; + +namespace pxr_sr = picsar::multi_physics::utils::serialization; +namespace pxr_bw = picsar::multi_physics::phys::breit_wheeler; +namespace pxr_qs = picsar::multi_physics::phys::quantum_sync; + +const auto line_commands = vector{ + {"-h" , ArgType::NoArg , "Prints all command line arguments"}, + {"--table" , ArgType::String , "Either BW (Breit-Wheeler) or QS (Quantum Synchrotron)"}, + {"--mode" , ArgType::String, "Precision of the calculations: either DP (double) or SP (single)"}, + {"--dndt_chi_min" , ArgType::Double , "minimum chi parameter for the dNdt table"}, + {"--dndt_chi_max" , ArgType::Double , "maximum chi parameter for the dNdt table"}, + {"--dndt_how_many" , ArgType::Integer, "number of points in the dNdt table"}, + {"--pair_chi_min" , ArgType::Double , "minimum chi for the pair production table (BW only)"}, + {"--pair_chi_max" , ArgType::Double , "maximum chi for the pair production table (BW only)"}, + {"--pair_chi_how_many" , ArgType::Integer, "number of chi points in the pair production table (BW only)"}, + {"--pair_frac_how_many", ArgType::Integer, "number of frac points in the pair production table (BW only)"}, + {"--em_chi_min" , ArgType::Double , "minimum chi for the photon emission table (QS only)"}, + {"--em_chi_max" , ArgType::Double , "maximum chi for the photon emission production table (QS only)"}, + {"--em_frac_min" , ArgType::Double , "minimum frac for the photon emission production table (QS only)"}, + {"--em_chi_how_many" , ArgType::Integer, "number of chi points in the photon emission table (QS only)"}, + {"--em_frac_how_many" , ArgType::Integer, "number of frac points in the photon emission table (QS only)"}, + {"-o" , ArgType::String , "filename to save the lookup table"} +}; + +void GenerateTable (const ParsedArgs& args); +template +void GenerateTableBW (const ParsedArgs& args, const string& outfile_name); +template +void GenerateTableQS (const ParsedArgs& args, const string& outfile_name); + +int main (int argc, char** argv) +{ + cout << "### QED Table Generator ###" << endl; + const auto args_map = ParseArgs(line_commands, argc, argv); + + if (args_map.empty() || Contains(args_map, "-h")){ + PrintHelp(line_commands); + } + else{ + GenerateTable(args_map); + } + + SuccessExit(); +} + + +void GenerateTable (const ParsedArgs& args) +{ + if (!Contains(args, "--table")) + AbortWithMessage("'--table' argument must be provided"); + + if (!Contains(args, "--mode")) + AbortWithMessage("'--mode' argument must be provided"); + + if (!Contains(args, "-o")) + AbortWithMessage("'-o' argument must be provided"); + + const auto which_table = GetVal(args.at("--table")); + const auto mode = GetVal(args.at("--mode")); + const auto outfile_name = GetVal(args.at("-o")); + + bool use_double = false; + + if (mode == "DP"s) + use_double = true; + else if (mode == "SP"s) + use_double = false; + else + AbortWithMessage("'--mode' must be eiter 'DP' or 'SP'"); + + if (which_table == "BW"s){ + if (use_double) + GenerateTableBW(args, outfile_name); + else + GenerateTableBW(args, outfile_name); + } + else if (which_table == "QS"s) + if (use_double) + GenerateTableQS(args, outfile_name); + else + GenerateTableQS(args, outfile_name); + else + AbortWithMessage("'--table' must be eiter 'QS' or 'BW'"); +} + +template +void GenerateTableBW (const ParsedArgs& args, const string& outfile_name) +{ + cout << " Generating BW table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || + !Contains(args, "--dndt_how_many") || !Contains(args, "--pair_chi_min") || + !Contains(args, "--pair_chi_max") || !Contains(args, "--pair_chi_how_many") || + !Contains(args, "--pair_frac_how_many")) + AbortWithMessage("All the BW table arguments must be provided (check with -h)"); + + const auto dndt_table_params = + pxr_bw::dndt_lookup_table_params{ + static_cast(GetVal(args.at("--dndt_chi_min"))), + static_cast(GetVal(args.at("--dndt_chi_max"))), + GetVal(args.at("--dndt_how_many")) + }; + + const auto pair_prod_table_params = + pxr_bw::pair_prod_lookup_table_params{ + static_cast(GetVal(args.at("--pair_chi_min"))), + static_cast(GetVal(args.at("--pair_chi_max"))), + GetVal(args.at("--pair_chi_how_many")), + GetVal(args.at("--pair_frac_how_many")) + }; + + std::cout << " Params: \n"; + std::cout << " - dndt_chi_min : " << dndt_table_params.chi_phot_min << "\n"; + std::cout << " - dndt_chi_max : " << dndt_table_params.chi_phot_max << "\n"; + std::cout << " - dndt_how_many : " << dndt_table_params.chi_phot_how_many << "\n"; + std::cout << " - pair_chi_min : " << pair_prod_table_params.chi_phot_min << "\n"; + std::cout << " - pair_chi_max : " << pair_prod_table_params.chi_phot_max << "\n"; + std::cout << " - pair_chi_how_many : " << pair_prod_table_params.chi_phot_how_many << "\n"; + std::cout << " - pair_frac_how_many : " << pair_prod_table_params.frac_how_many << "\n"; + std::cout << " ----------------------- " << "\n\n"; + + auto dndt_table = + pxr_bw::dndt_lookup_table>{ + dndt_table_params}; + + auto pair_prod_table = + pxr_bw::pair_prod_lookup_table>{ + pair_prod_table_params}; + + dndt_table.generate(true); //Progress bar is displayed + pair_prod_table.generate(true); //Progress bar is displayed + + const auto data_dndt = dndt_table.serialize(); + const auto data_pair_prod = pair_prod_table.serialize(); + + const uint64_t size_first = data_dndt.size(); + + vector res{}; + pxr_sr::put_in(size_first, res); + for (const auto& tmp : data_dndt) + pxr_sr::put_in(tmp, res); + for (const auto& tmp : data_pair_prod) + pxr_sr::put_in(tmp, res); + + auto of = std::ofstream{outfile_name, std::ios_base::binary}; + of.write(res.data(), res.size()); + of.close(); + + cout << " Done! \n"; +} + +template +void GenerateTableQS (const ParsedArgs& args, const string& outfile_name) +{ + cout << " Generating QS table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || + !Contains(args, "--dndt_how_many") || !Contains(args, "--em_chi_min") || + !Contains(args, "--em_chi_max") || !Contains(args, "--em_frac_min") || + !Contains(args, "--em_chi_how_many") || !Contains(args, "--em_frac_how_many")) + AbortWithMessage("All the QS table arguments must be provided (check with -h)"); + + const auto dndt_table_params = + pxr_qs::dndt_lookup_table_params{ + static_cast(GetVal(args.at("--dndt_chi_min"))), + static_cast(GetVal(args.at("--dndt_chi_max"))), + GetVal(args.at("--dndt_how_many")) + }; + + const auto phot_em_table_params = + pxr_qs::photon_emission_lookup_table_params{ + static_cast(GetVal(args.at("--em_chi_min"))), + static_cast(GetVal(args.at("--em_chi_max"))), + static_cast(GetVal(args.at("--em_frac_min"))), + GetVal(args.at("--em_chi_how_many")), + GetVal(args.at("--em_frac_how_many")) + }; + + std::cout << " Params: \n"; + std::cout << " - dndt_chi_min : " << dndt_table_params.chi_part_min << "\n"; + std::cout << " - dndt_chi_max : " << dndt_table_params.chi_part_max << "\n"; + std::cout << " - dndt_how_many : " << dndt_table_params.chi_part_how_many << "\n"; + std::cout << " - phot_em_chi_min : " << phot_em_table_params.chi_part_min << "\n"; + std::cout << " - phot_em_chi_max : " << phot_em_table_params.chi_part_max << "\n"; + std::cout << " - phot_em_frac_min : " << phot_em_table_params.frac_min << "\n"; + std::cout << " - phot_em_chi_how_many : " << phot_em_table_params.chi_part_how_many << "\n"; + std::cout << " - phot_em_frac_how_many : " << phot_em_table_params.frac_how_many << "\n"; + std::cout << " ----------------------- " << "\n\n"; + + auto dndt_table = + pxr_qs::dndt_lookup_table>{ + dndt_table_params}; + + auto phot_em_table = + pxr_qs::photon_emission_lookup_table>{ + phot_em_table_params}; + + dndt_table.generate(true); //Progress bar is displayed + phot_em_table.generate(true); //Progress bar is displayed + + const auto data_dndt = dndt_table.serialize(); + const auto data_phot_em = phot_em_table.serialize(); + + const uint64_t size_first = data_dndt.size(); + + vector res{}; + pxr_sr::put_in(size_first, res); + for (const auto& tmp : data_dndt) + pxr_sr::put_in(tmp, res); + for (const auto& tmp : data_phot_em) + pxr_sr::put_in(tmp, res); + + auto of = std::ofstream{outfile_name, std::ios_base::binary}; + of.write(res.data(), res.size()); + of.close(); + + cout << " Done! \n"; +} diff --git a/Tools/QedTablesUtils/Source/QedTableReader.cpp b/Tools/QedTablesUtils/Source/QedTableReader.cpp new file mode 100644 index 00000000000..ae689a87643 --- /dev/null +++ b/Tools/QedTablesUtils/Source/QedTableReader.cpp @@ -0,0 +1,240 @@ +#include "QedTableCommons.H" +#include "ArgParser/QedTablesArgParser.H" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ArgParser; +using namespace std; + +namespace pxr_sr = picsar::multi_physics::utils::serialization; +namespace pxr_bw = picsar::multi_physics::phys::breit_wheeler; +namespace pxr_qs = picsar::multi_physics::phys::quantum_sync; + +const auto line_commands = vector{ + {"-h" , ArgType::NoArg , "Prints all command line arguments"}, + {"-i" , ArgType::String , "Name of the file to open"}, + {"--table" , ArgType::String , "Either BW (Breit-Wheeler) or QS (Quantum Synchrotron)"}, + {"--mode" , ArgType::String , "Precision of the calculations: either DP (double) or SP (single)"}, + {"-o" , ArgType::String , "filename to save the lookup table in human-readable format"} +}; + +void ReadTable (const ParsedArgs& args); +template +void ReadTableBW (const string& input_file, const string& outfile_name); +template +void ReadTableQS ( + const string& input_file, const string& outfile_name); + +/*Wrapper class to access protected data*/ +template +class bw_pair_production_table_wrapper : + public pxr_bw::pair_prod_lookup_table> +{ + public: + void write_table_data(std::ofstream& of) const; +}; + +/*Wrapper class to access protected data*/ +template +class qs_photon_emission_table_wrapper : + public pxr_qs::photon_emission_lookup_table> +{ + public: + void write_table_data(std::ofstream& of) const; +}; + +int main (int argc, char** argv) +{ + cout << "### QED Table Reader ###" << endl; + const auto args_map = ParseArgs(line_commands, argc, argv); + + if (args_map.empty() || Contains(args_map, "-h")){ + PrintHelp(line_commands); + } + else{ + ReadTable(args_map); + } + + SuccessExit(); +} + +void ReadTable (const ParsedArgs& args) +{ + if (!Contains(args, "--table")) + AbortWithMessage("'--table' argument must be provided"); + + if (!Contains(args, "--mode")) + AbortWithMessage("'--mode' argument must be provided"); + + if (!Contains(args, "-o")) + AbortWithMessage("'-o' argument must be provided"); + + if (!Contains(args, "-i")) + AbortWithMessage("'-i' argument must be provided"); + + const auto which_table = GetVal(args.at("--table")); + const auto mode = GetVal(args.at("--mode")); + const auto input_file = GetVal(args.at("-i")); + const auto outfile_name = GetVal(args.at("-o")); + + bool use_double = false; + + if (mode == "DP"s) + use_double = true; + else if (mode == "SP"s) + use_double = false; + else + AbortWithMessage("'--mode' must be eiter 'DP' or 'SP'"); + + if (which_table == "BW"s){ + if (use_double) + ReadTableBW(input_file, outfile_name); + else + ReadTableBW(input_file, outfile_name); + } + else if (which_table == "QS"s) + if (use_double) + ReadTableQS(input_file, outfile_name); + else + ReadTableQS(input_file, outfile_name); + else + AbortWithMessage("'--table' must be eiter 'QS' or 'BW'"); +} + +template +void ReadTableBW (const string& input_file, const string& outfile_name) +{ + cout << " Reading BW table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + auto ifs = ifstream(input_file, std::ios::binary); + auto raw_data = vector(istreambuf_iterator(ifs), {}); + ifs.close(); + + auto raw_iter = raw_data.begin(); + const auto size_first = pxr_sr::get_out(raw_iter); + if(size_first <= 0 || size_first >= raw_data.size() ) + AbortWithMessage("Something is wrong with " + input_file); + + const auto raw_dndt_table = vector{ + raw_iter, raw_iter+size_first}; + + const auto raw_pair_prod_table = vector{ + raw_iter+size_first, raw_data.end()}; + + const auto dndt_table = + pxr_bw::dndt_lookup_table>{raw_dndt_table}; + const auto pair_prod_table = + pxr_bw::pair_prod_lookup_table>{raw_pair_prod_table}; + + if (!dndt_table.is_init() || !pair_prod_table.is_init()) + AbortWithMessage("Something went wrong with lookup table initialization"); + + auto of_dndt = ofstream{outfile_name + "_dndt"}; + of_dndt << std::setprecision(std::numeric_limits::digits10 + 1); + const auto coord_dndt = dndt_table.get_all_coordinates(); + for (const auto& cc : coord_dndt ) + of_dndt << cc << " " << dndt_table.interp(cc) << "\n"; + of_dndt.close(); + + auto of_pair = ofstream{outfile_name + "_pair"}; + of_pair << std::setprecision(std::numeric_limits::digits10 + 1); + const auto wrapper = bw_pair_production_table_wrapper{pair_prod_table}; + wrapper.write_table_data(of_pair); + of_pair.close(); +} + +template +void ReadTableQS ( + const string& input_file, const string& outfile_name) +{ + cout << " Reading QS table " << + (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + + auto ifs = ifstream(input_file, std::ios::binary); + auto raw_data = vector(istreambuf_iterator(ifs), {}); + ifs.close(); + + auto raw_iter = raw_data.begin(); + const auto size_first = pxr_sr::get_out(raw_iter); + if(size_first <= 0 || size_first >= raw_data.size() ) + AbortWithMessage("Something is wrong with " + input_file); + + const auto raw_dndt_table = vector{ + raw_iter, raw_iter+size_first}; + + const auto raw_phot_em_table = vector{ + raw_iter+size_first, raw_data.end()}; + + const auto dndt_table = + pxr_qs::dndt_lookup_table>{raw_dndt_table}; + const auto phot_em_table = + pxr_qs::photon_emission_lookup_table>{raw_phot_em_table}; + + if (!dndt_table.is_init() || !phot_em_table.is_init()) + AbortWithMessage("Something went wrong with lookup table initialization"); + + auto of_dndt = ofstream{outfile_name + "_dndt"}; + of_dndt << std::setprecision(std::numeric_limits::digits10 + 1); + const auto coord_dndt = dndt_table.get_all_coordinates(); + for (const auto& cc : coord_dndt ) + of_dndt << cc << " " << dndt_table.interp(cc) << "\n"; + of_dndt.close(); + + auto of_phot_em = ofstream{outfile_name + "_phot_em"}; + of_phot_em << std::setprecision(std::numeric_limits::digits10 + 1); + + const auto wrapper = qs_photon_emission_table_wrapper{phot_em_table}; + wrapper.write_table_data(of_phot_em); + of_phot_em.close(); + + return; +} + +template +void +bw_pair_production_table_wrapper::write_table_data( + std::ofstream& of) const +{ + const auto how_many_x = this->m_table.get_how_many_x(); + const auto how_many_y = this->m_table.get_how_many_y(); + for (int i = 0; i < how_many_x; ++i){ + for (int j = 0; j < how_many_y; ++j){ + const auto xcoord = this->m_table.get_x_coord(i); + const auto ycoord = this->m_table.get_y_coord(j); + const auto val = this->m_table.get_val(i,j); + of << std::exp(xcoord) << " " << ycoord*std::exp(xcoord) + << " " << val << "\n"; + } + } +} + +template +void +qs_photon_emission_table_wrapper::write_table_data( + std::ofstream& of) const +{ + const auto how_many_x = this->m_table.get_how_many_x(); + const auto how_many_y = this->m_table.get_how_many_y(); + for (int i = 0; i < how_many_x; ++i){ + for (int j = 0; j < how_many_y; ++j){ + const auto xcoord = this->m_table.get_x_coord(i); + const auto ycoord = this->m_table.get_y_coord(j); + const auto val = this->m_table.get_val(i,j); + of << std::exp(xcoord) << " " << std::exp(ycoord)*std::exp(xcoord) + << " " << std::exp(val) << "\n"; + } + } +} diff --git a/Tools/Release/updateAMReX.py b/Tools/Release/updateAMReX.py index 4f2d84e4eb7..9dfa7fbeb41 100755 --- a/Tools/Release/updateAMReX.py +++ b/Tools/Release/updateAMReX.py @@ -9,9 +9,9 @@ # when building WarpX. # import datetime -from pathlib import Path import re import sys +from pathlib import Path import requests diff --git a/Tools/Release/updatePICSAR.py b/Tools/Release/updatePICSAR.py index ce1efec0a99..7e61679d371 100755 --- a/Tools/Release/updatePICSAR.py +++ b/Tools/Release/updatePICSAR.py @@ -9,9 +9,9 @@ # when building WarpX. # import datetime -from pathlib import Path import re import sys +from pathlib import Path import requests diff --git a/Tools/Release/updatepyAMReX.py b/Tools/Release/updatepyAMReX.py index 8ba10c895e0..500781e0880 100755 --- a/Tools/Release/updatepyAMReX.py +++ b/Tools/Release/updatepyAMReX.py @@ -9,9 +9,9 @@ # when building WarpX. # import datetime -from pathlib import Path import re import sys +from pathlib import Path import requests diff --git a/Tools/machines/adastra-cines/submit.sh b/Tools/machines/adastra-cines/submit.sh index 15a2b292b58..e8bcb19975e 100644 --- a/Tools/machines/adastra-cines/submit.sh +++ b/Tools/machines/adastra-cines/submit.sh @@ -31,6 +31,14 @@ export MPICH_GPU_SUPPORT_ENABLED=1 # or, less invasive: export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor +# note +# On machines with similar architectures (Frontier, OLCF) these settings +# seem to prevent the following issue: +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors +export MPICH_SMP_SINGLE_COPY_MODE=NONE +export FI_CXI_RX_MATCH_MODE=software + # note # this environment setting is needed to avoid that rocFFT writes a cache in # the home directory, which does not scale. diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 9460f9a5175..98c30cfca8f 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -14,7 +14,7 @@ set -eu -o pipefail # Check: ###################################################################### # -# Was perlmutter_gpu_warpx.profile sourced and configured correctly? +# Was frontier_warpx.profile sourced and configured correctly? if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your frontier_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi diff --git a/Tools/machines/frontier-olcf/submit.sh b/Tools/machines/frontier-olcf/submit.sh index 13d981238a3..b340d56546e 100644 --- a/Tools/machines/frontier-olcf/submit.sh +++ b/Tools/machines/frontier-olcf/submit.sh @@ -32,7 +32,7 @@ export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # Seen since August 2023 -# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors # https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors export MPICH_SMP_SINGLE_COPY_MODE=NONE export FI_CXI_RX_MATCH_MODE=software diff --git a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example index a430579a7d5..caf85983b58 100644 --- a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example +++ b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example @@ -10,7 +10,7 @@ export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH spack load ninja@1.11.1%fj@4.10.0 # optional: for PSATD -spack load fujitsu-fftw +spack load fujitsu-fftw@1.1.0%fj@4.10.0 # optional: for QED lookup table generation support spack load boost@1.80.0%fj@4.8.1/zc5pwgc diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch new file mode 100644 index 00000000000..0353c08456f --- /dev/null +++ b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch @@ -0,0 +1,38 @@ +#!/bin/bash -l + +# Copyright 2024 The WarpX Community +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +#SBATCH -t 00:10:00 +#SBATCH -N 1 +#SBATCH -J WarpX +#SBATCH -A +#SBATCH --partition=gpu +#SBATCH --exclusive +#SBATCH --ntasks-per-node=2 +#SBATCH --cpus-per-task=20 +#SBATCH --gpus-per-task=v100:1 +#SBATCH --gpu-bind=single:1 +#SBATCH -o WarpX.o%j +#SBATCH -e WarpX.e%j + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=inputs + +# threads for OpenMP and threaded compressors per MPI rank +# per node are 2x 2.4 GHz Intel Xeon Gold 6148 +# note: the system seems to only expose cores (20 per socket), +# not hyperthreads (40 per socket) +export SRUN_CPUS_PER_TASK=20 +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} + +# GPU-aware MPI optimizations +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# run WarpX +srun --cpu-bind=cores \ + ${EXE} ${INPUTS} ${GPU_AWARE_MPI} \ + > output.txt diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example new file mode 100644 index 00000000000..98476eecfab --- /dev/null +++ b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example @@ -0,0 +1,62 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi + +# required dependencies +module purge +module load gcc/10.3.0 +module load cuda/12.1.1 +module load cmake/3.26.3 +module load openblas/0.3.23 +module load openmpi/4.1.6-cuda + +# optional: for QED support +module load boost/1.78.0 + +# optional: for openPMD and PSATD+RZ support +module load phdf5/1.12.1 + +SW_DIR="${HOME}/sw/greatlakes/v100" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc2-2.14.4:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.0:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc2-2.14.4/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.0/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH + +export PATH=${SW_DIR}/adios2-2.10.0/bin:${PATH} + +# optional: for Python bindings or libEnsemble +module load python/3.12.1 + +if [ -d "${SW_DIR}/venvs/warpx-v100" ] +then + source ${SW_DIR}/venvs/warpx-v100/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --partition=gpu --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --partition=gpu --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" + +# optimize CUDA compilation for V100 +export AMREX_CUDA_ARCH=7.0 + +# optimize CPU microarchitecture for Intel Xeon Gold 6148 +export CXXFLAGS="-march=skylake-avx512" +export CFLAGS="-march=skylake-avx512" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh new file mode 100755 index 00000000000..9361fd6c64b --- /dev/null +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was greatlakes_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your greatlakes_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +echo "Cleaning up prior installation directory... This may take several minutes." +SW_DIR="${HOME}/sw/greatlakes/v100" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d $HOME/src/c-blosc2 ] +then + cd $HOME/src/c-blosc2 + git fetch --prune + git checkout v2.14.4 + cd - +else + git clone -b v2.14.4 https://github.com/Blosc/c-blosc2.git $HOME/src/c-blosc2 +fi +rm -rf $HOME/src/c-blosc2-v100-build +cmake -S $HOME/src/c-blosc2 -B ${build_dir}/c-blosc2-v100-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_FUZZERS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc2-2.14.4 +cmake --build ${build_dir}/c-blosc2-v100-build --target install --parallel 8 +rm -rf ${build_dir}/c-blosc2-v100-build + +# ADIOS2 +if [ -d $HOME/src/adios2 ] +then + cd $HOME/src/adios2 + git fetch --prune + git checkout v2.10.0 + cd - +else + git clone -b v2.10.0 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 +fi +rm -rf $HOME/src/adios2-v100-build +cmake \ + -S $HOME/src/adios2 \ + -B ${build_dir}/adios2-v100-build \ + -DADIOS2_USE_Blosc2=ON \ + -DADIOS2_USE_Campaign=OFF \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 +cmake --build ${build_dir}/adios2-v100-build --target install -j 8 +rm -rf ${build_dir}/adios2-v100-build + +# BLAS++ (for PSATD+RZ) +if [ -d $HOME/src/blaspp ] +then + cd $HOME/src/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp +fi +rm -rf $HOME/src/blaspp-v100-build +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-v100-build --target install --parallel 8 +rm -rf ${build_dir}/blaspp-v100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d $HOME/src/lapackpp ] +then + cd $HOME/src/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp +fi +rm -rf $HOME/src/lapackpp-v100-build +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-v100-build --target install --parallel 8 +rm -rf ${build_dir}/lapackpp-v100-build + + +# Python ###################################################################### +# +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade virtualenv +python3 -m pip cache purge +rm -rf ${SW_DIR}/venvs/warpx-v100 +python3 -m venv ${SW_DIR}/venvs/warpx-v100 +source ${SW_DIR}/venvs/warpx-v100/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies +python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch # CUDA 12 compatible wheel +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index a2b9c7b3855..51d4bcfe2c5 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -70,7 +70,7 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/adios2-pm-gpu-build diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 0e466f1f57f..814e134bad4 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -14,15 +14,17 @@ set -eu -o pipefail # Check: ###################################################################### # -# Was perlmutter_gpu_warpx.profile sourced and configured correctly? +# Was lumi_warpx.profile sourced and configured correctly? if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lumi_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi # Remove old dependencies ##################################################### # +SRC_DIR="${HOME}/src" SW_DIR="${HOME}/sw/lumi/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} # remove common user mistakes in python, located in .local instead of a venv python3 -m pip uninstall -qq -y pywarpx @@ -33,63 +35,113 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] +if [ -d ${SRC_DIR}/blaspp ] then - cd $HOME/src/blaspp + cd ${SRC_DIR}/blaspp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -rm -rf $HOME/src/blaspp-lumi-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-lumi-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-lumi-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-lumi-gpu-build +rm -rf ${build_dir}/blaspp-lumi-gpu-build +CXX=$(which CC) \ +cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-lumi-gpu-build \ + -Duse_openmp=OFF \ + -Dgpu_backend=hip \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-lumi-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-lumi-gpu-build # LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] +if [ -d ${SRC_DIR}/lapackpp ] then - cd $HOME/src/lapackpp + cd ${SRC_DIR}/lapackpp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -rm -rf $HOME/src/lapackpp-lumi-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-lumi-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-lumi-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-lumi-gpu-build +rm -rf ${build_dir}/lapackpp-lumi-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" \ +cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-lumi-gpu-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-lumi-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-lumi-gpu-build -# c-blosc (I/O compression, for OpenPMD) -if [ -d $HOME/src/c-blosc ] +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ] then - # git repository is already there - : + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc fi -rm -rf $HOME/src/c-blosc-lu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-lu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-lu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-lu-build +rm -rf ${build_dir}/c-blosc-lu-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-lu-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-lu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-lu-build -# ADIOS2 v. 2.8.3 (for OpenPMD) -if [ -d $HOME/src/adios2 ] +# HDF5 (for openPMD) +if [ -d ${SRC_DIR}/hdf5 ] then - # git repository is already there - : + cd ${SRC_DIR}/hdf5 + git fetch --prune + git checkout hdf5-1_14_1-2 + cd - +else + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 +fi +rm -rf ${build_dir}/hdf5-lu-build +cmake -S ${SRC_DIR}/hdf5 \ + -B ${build_dir}/hdf5-lu-build \ + -DBUILD_TESTING=OFF \ + -DHDF5_ENABLE_PARALLEL=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 +cmake --build ${build_dir}/hdf5-lu-build --target install --parallel 10 +rm -rf ${build_dir}/hdf5-lu-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 fi -rm -rf $HOME/src/adios2-lu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-lu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/adios2-2.8.3 -cmake --build $HOME/src/adios2-lu-build --target install -j 16 -rm -rf $HOME/src/adios2-lu-build +rm -rf ${build_dir}/adios2-lu-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-lu-build \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_HDF5=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/adios2-2.8.3 +cmake --build ${build_dir}/adios2-lu-build --target install -j 16 +rm -rf ${build_dir}/adios2-lu-build # Python ###################################################################### @@ -114,9 +166,9 @@ python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt # optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +python3 -m pip install -r ${SRC_DIR}/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 -#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/lumi-csc/submit.sh b/Tools/machines/lumi-csc/lumi.sbatch similarity index 97% rename from Tools/machines/lumi-csc/submit.sh rename to Tools/machines/lumi-csc/lumi.sbatch index d784471acd5..6d4ce1ee37f 100644 --- a/Tools/machines/lumi-csc/submit.sh +++ b/Tools/machines/lumi-csc/lumi.sbatch @@ -20,7 +20,7 @@ date export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # Seen since August 2023 seen on OLCF (not yet seen on LUMI?) -# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors # https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors #export MPICH_SMP_SINGLE_COPY_MODE=NONE #export FI_CXI_RX_MATCH_MODE=software diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 2cb44035ce4..33aff1946c2 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -1,5 +1,5 @@ # please set your project account -#export proj= +#export proj="project_..." # required dependencies module load LUMI/23.09 partition/G @@ -10,23 +10,30 @@ module load buildtools/23.09 module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +SW_DIR="${HOME}/sw/lumi/gpu" +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load Boost/1.82.0-cpeCray-23.09 # optional: for openPMD support -module load cray-hdf5/1.12.2.7 -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble module load cray-python/3.10.10 +if [ -d "${SW_DIR}/venvs/warpx-lumi" ] +then + source ${SW_DIR}/venvs/warpx-lumi/bin/activate +fi + # an alias to request an interactive batch node for one hour # for paralle execution, start on the batch node: srun alias getNode="salloc -A $proj -J warpx -t 01:00:00 -p dev-g -N 1 --ntasks-per-node=8 --gpus-per-task=1 --gpu-bind=closest" @@ -50,4 +57,4 @@ export CXX=$(which amdclang++) export FC=$(which amdflang) export CFLAGS="-I${ROCM_PATH}/include" export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" +export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index dbe59438a16..f65e43891d0 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -44,6 +44,9 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -55,9 +58,9 @@ else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi rm -rf $HOME/src/c-blosc-pm-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-pm-cpu-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pm-cpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -70,9 +73,9 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-cpu-build --target install -j 16 -rm -rf $HOME/src/adios2-pm-cpu-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j 16 +rm -rf ${build_dir}/adios2-pm-cpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -86,9 +89,9 @@ else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-cpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-pm-cpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pm-cpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -102,9 +105,9 @@ else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-cpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-pm-cpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pm-cpu-build # Python ###################################################################### @@ -112,9 +115,9 @@ rm -rf $HOME/src/lapackpp-pm-cpu-build python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv ${SW_DIR}/venvs/warpx -source ${SW_DIR}/venvs/warpx/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-cpu +python3 -m venv ${SW_DIR}/venvs/warpx-cpu +source ${SW_DIR}/venvs/warpx-cpu/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,7 +131,12 @@ MPICC="cc -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-b python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard +# install or update WarpX dependencies python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 3fe76359953..9ac5800c6ce 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -44,6 +44,9 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -55,9 +58,9 @@ else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi rm -rf $HOME/src/c-blosc-pm-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-pm-gpu-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pm-gpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -70,9 +73,9 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-gpu-build --target install -j 16 -rm -rf $HOME/src/adios2-pm-gpu-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j 16 +rm -rf ${build_dir}/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -86,9 +89,9 @@ else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-pm-gpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -102,9 +105,9 @@ else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-pm-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pm-gpu-build # Python ###################################################################### @@ -112,9 +115,9 @@ rm -rf $HOME/src/lapackpp-pm-gpu-build python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv ${SW_DIR}/venvs/warpx -source ${SW_DIR}/venvs/warpx/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-gpu +python3 -m venv ${SW_DIR}/venvs/warpx-gpu +source ${SW_DIR}/venvs/warpx-gpu/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,11 +131,13 @@ MPICC="cc -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4p python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard +# install or update WarpX dependencies python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -python3 -m pip install cupy-cuda12x # CUDA 12 compatible wheel -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) python3 -m pip install --upgrade torch # CUDA 12 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch index a1687b2a3b4..d13c7e3b4e5 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch @@ -33,6 +33,7 @@ INPUTS=inputs_small export SRUN_CPUS_PER_TASK=16 # 8 cores per chiplet, 2x SMP export OMP_PLACES=threads export OMP_PROC_BIND=spread +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} srun --cpu-bind=cores \ ${EXE} ${INPUTS} \ diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 67ae36c9f1a..1b0ac3182d5 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -33,9 +33,9 @@ export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/ # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx" ] +if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu" ] then - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx/bin/activate + source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate fi # an alias to request an interactive batch node for one hour diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch index 873fd30355f..f2ea5fa3e7f 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch @@ -34,6 +34,7 @@ export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank # note: 16 avoids hyperthreading (32 virtual cores, 16 physical) export SRUN_CPUS_PER_TASK=16 +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 7bf5dd13116..759df0b923a 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -37,9 +37,9 @@ export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/ # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx" ] +if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu" ] then - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx/bin/activate + source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate fi # an alias to request an interactive batch node for one hour diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 03225878700..5a2c0916432 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -17,7 +17,6 @@ set(WarpX_EB @WarpX_EB@) set(WarpX_EB_FOUND ${WarpX_EB}) set(WarpX_LIB @WarpX_LIB@) set(WarpX_LIB_FOUND ${WarpX_LIB}) -WarpX_GPUCLOCK # dependencies set(WarpX_MPI @WarpX_MPI@) diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index a189a8649d7..9b9feb64437 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -444,7 +444,6 @@ function(warpx_print_summary) message(" COMPUTE: ${WarpX_COMPUTE}") message(" DIMS: ${WarpX_DIMS}") message(" Embedded Boundary: ${WarpX_EB}") - message(" GPU clock timers: ${WarpX_GPUCLOCK}") message(" IPO/LTO: ${WarpX_IPO}") message(" LIB: ${WarpX_LIB}${LIB_TYPE}") message(" MPI: ${WarpX_MPI}") @@ -461,6 +460,7 @@ function(warpx_print_summary) message(" OPENPMD: ${WarpX_OPENPMD}") message(" QED: ${WarpX_QED}") message(" QED table generation: ${WarpX_QED_TABLE_GEN}") + message(" QED tools: ${WarpX_QED_TOOLS}") message(" SENSEI: ${WarpX_SENSEI}") message("") endfunction() diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index a1045457ec0..349a4fd3f63 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.03 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.05 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.03" +set(WarpX_amrex_branch "7ca419ebb90da60fefc01d8c1816846fff8638a5" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake index a1e93a9e33c..9086421356f 100644 --- a/cmake/dependencies/PICSAR.cmake +++ b/cmake/dependencies/PICSAR.cmake @@ -14,18 +14,27 @@ function(find_picsar) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) # Enable or disable QED lookup tables generation - - # If table generation is enabled, enable or disable - # openMP support depending on WarpX_COMPUTE - if(WarpX_QED_TABLE_GEN) + if(WarpX_QED_TABLE_GEN OR WarpX_QED_TOOLS) set(PXRMP_QED_TABLEGEN ON CACHE INTERNAL "") - if(WarpX_COMPUTE STREQUAL OMP) - set(PXRMP_QED_OMP ON CACHE INTERNAL "") + else() + set(PXRMP_QED_TABLEGEN OFF CACHE INTERNAL "") + endif() + + # Enable or disable OpenMP for lookup tables generation + if(PXRMP_QED_TABLEGEN) + if(WarpX_QED_TABLES_GEN_OMP STREQUAL AUTO) + find_package(OpenMP REQUIRED CXX) + if(OpenMP_CXX_FOUND) + set(PXRMP_QED_OMP ON CACHE INTERNAL "") + else() + set(PXRMP_QED_OMP OFF CACHE INTERNAL "") + endif() + elseif(WarpX_QED_TABLES_GEN_OMP STREQUAL ON) + set(PXRMP_QED_OMP ON CACHE INTERNAL "") else() - set(PXRMP_QED_OMP OFF CACHE INTERNAL "") + set(PXRMP_QED_OMP OFF CACHE INTERNAL "") endif() else() - set(PXRMP_QED_TABLEGEN OFF CACHE INTERNAL "") set(PXRMP_QED_OMP OFF CACHE INTERNAL "") endif() diff --git a/cmake/dependencies/openPMD.cmake b/cmake/dependencies/openPMD.cmake index 93599d7505e..f58d37ee92e 100644 --- a/cmake/dependencies/openPMD.cmake +++ b/cmake/dependencies/openPMD.cmake @@ -13,7 +13,7 @@ function(find_openpmd) if(WarpX_openpmd_internal OR WarpX_openpmd_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - # see https://openpmd-api.readthedocs.io/en/0.15.1/dev/buildoptions.html + # see https://openpmd-api.readthedocs.io/en/0.15.2/dev/buildoptions.html set(openPMD_USE_ADIOS1 OFF CACHE INTERNAL "") set(openPMD_USE_MPI ${WarpX_MPI} CACHE INTERNAL "") set(openPMD_USE_PYTHON OFF CACHE INTERNAL "") @@ -92,7 +92,7 @@ if(WarpX_OPENPMD) set(WarpX_openpmd_repo "https://github.com/openPMD/openPMD-api.git" CACHE STRING "Repository URI to pull and build openPMD-api from if(WarpX_openpmd_internal)") - set(WarpX_openpmd_branch "0.15.1" + set(WarpX_openpmd_branch "0.15.2" CACHE STRING "Repository branch for WarpX_openpmd_repo if(WarpX_openpmd_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 6360da84276..79feddf3184 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.03 CONFIG REQUIRED) + find_package(pyAMReX 24.05 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.03" +set(WarpX_pyamrex_branch "1b795c2c9741c8a63a362ceef8bd5e70cb242e92" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index e65cd206f34..0a7ec260493 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.11.1 CONFIG REQUIRED) + find_package(pybind11 2.12.0 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.11.1" +set(WarpX_pybind11_branch "v2.12.0" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") diff --git a/pyproject.toml b/pyproject.toml index f53522b5622..f9e615f9b83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,7 @@ requires = [ "packaging>=23", ] build-backend = "setuptools.build_meta" + +[tool.isort] +known_first_party = ["amrex", "picmistandard", "pywarpx", "warpx"] +profile = "black" diff --git a/run_test.sh b/run_test.sh index f556dd37e76..099a59702a2 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.03 && cd - +cd amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets @@ -80,6 +80,8 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git +# FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 +cd regression_testing && git checkout 93ddfb11456f47d6555c39388ba1a4ead61fbf4e && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks diff --git a/setup.py b/setup.py index b0f919d707a..164f4f1395f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.03', + version = '24.05', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.',