Merge remote-tracking branch 'mainline/development' into RecordingDat…

…aAtStation
RevathiJambunathan · Jan 31, 2024 · 62c8c40 · 62c8c40
2 parents 15e810a + fa422b5
commit 62c8c40
Show file tree

Hide file tree

Showing 11 changed files with 77 additions and 51 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -115,7 +115,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd -
+        cd ../amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
         ccache -s

diff --git a/Docs/source/theory/intro.rst b/Docs/source/theory/intro.rst
@@ -8,8 +8,7 @@ Introduction
 
    Plasma laser-driven (top) and charged-particles-driven (bottom) acceleration (rendering from 3-D Particle-In-Cell simulations). A laser beam (red and blue disks in top picture) or a charged particle beam (red dots in bottom picture) propagating (from left to right) through an under-dense plasma (not represented) displaces electrons, creating a plasma wakefield that supports very high electric fields (pale blue and yellow). These electric fields, which can be orders of magnitude larger than with conventional techniques, can be used to accelerate a short charged particle beam (white) to high-energy over a very short distance.
 
-Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`, with
-accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play.
+Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`. Accurate modeling of wake formation, electron self-trapping and acceleration require fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play.
 
 For such simulations,
 the most popular algorithm is the Particle-In-Cell (or PIC) technique,

diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py
@@ -1575,7 +1575,7 @@ def __init__(self, name, species, scattering_processes, ndt=None, **kw):
 
         self.handle_init(kw)
 
-    def initialize_inputs(self):
+    def collision_initialize_inputs(self):
         collision = pywarpx.Collisions.newcollision(self.name)
         collision.type = 'dsmc'
         collision.species = [species.name for species in self.species]

diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini
@@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more
 
 [AMReX]
 dir = /home/regtester/git/amrex/
-branch = 73b215557c0e842c3e829b683939bbb7a7e12373
+branch = 689144d157a0106faf3d0ae89f8d90b0250cf975
 
 [source]
 dir = /home/regtester/git/WarpX

diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini
@@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det
 
 [AMReX]
 dir = /home/regtester/AMReX_RegTesting/amrex/
-branch = 73b215557c0e842c3e829b683939bbb7a7e12373
+branch = 689144d157a0106faf3d0ae89f8d90b0250cf975
 
 [source]
 dir = /home/regtester/AMReX_RegTesting/warpx
@@ -3021,24 +3021,24 @@ doVis = 0
 compareParticles = 0
 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py
 
-# [Python_background_mcc_1d_tridiag]
-# buildDir = .
-# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py
-# runtime_params =
-# customRunCmd = python3 PICMI_inputs_1d.py --test
-# dim = 1
-# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE
-# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF
-# target = pip_install
-# restartTest = 0
-# useMPI = 1
-# numprocs = 2
-# useOMP = 1
-# numthreads = 1
-# compileTest = 0
-# doVis = 0
-# compareParticles = 0
-# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py
+[Python_background_mcc_1d_tridiag]
+buildDir = .
+inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py
+runtime_params =
+customRunCmd = python3 PICMI_inputs_1d.py --test
+dim = 1
+addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE
+cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF
+target = pip_install
+restartTest = 0
+useMPI = 1
+numprocs = 2
+useOMP = 1
+numthreads = 1
+compileTest = 0
+doVis = 0
+compareParticles = 0
+analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py
 
 [Python_collisionXZ]
 buildDir = .
@@ -3079,24 +3079,24 @@ doVis = 0
 compareParticles = 0
 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py
 
-# [Python_dsmc_1d]
-# buildDir = .
-# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py
-# runtime_params =
-# customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc
-# dim = 1
-# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE
-# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF
-# target = pip_install
-# restartTest = 0
-# useMPI = 1
-# numprocs = 2
-# useOMP = 1
-# numthreads = 1
-# compileTest = 0
-# doVis = 0
-# compareParticles = 0
-# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py
+[Python_dsmc_1d]
+buildDir = .
+inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py
+runtime_params =
+customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc
+dim = 1
+addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE
+cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF
+target = pip_install
+restartTest = 0
+useMPI = 1
+numprocs = 2
+useOMP = 1
+numthreads = 1
+compileTest = 0
+doVis = 0
+compareParticles = 0
+analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py
 
 [Python_ElectrostaticSphereEB]
 buildDir = .

diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp
@@ -39,6 +39,10 @@
 #include "Utils/WarpXAlgorithmSelection.H"
 #include "Utils/WarpXConst.H"
 #include "Utils/WarpXProfilerWrapper.H"
+#ifdef AMREX_USE_EB
+#   include "EmbeddedBoundary/ParticleBoundaryProcess.H"
+#   include "EmbeddedBoundary/ParticleScraper.H"
+#endif
 #include "WarpX.H"
 
 #include <ablastr/warn_manager/WarnManager.H>
@@ -1437,6 +1441,12 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int
         }
     }
 
+    // Remove particles that are inside the embedded boundaries
+#ifdef AMREX_USE_EB
+    auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB();
+    scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb());
+#endif
+
     // The function that calls this is responsible for redistributing particles.
 }
 
@@ -1930,6 +1940,12 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector,
         }
     }
 
+    // Remove particles that are inside the embedded boundaries
+#ifdef AMREX_USE_EB
+    auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB();
+    scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb());
+#endif
+
     // Redistribute the new particles that were added to the temporary container.
     // (This eliminates invalid particles, and makes sure that particles
     // are in the right tile.)
@@ -2864,7 +2880,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti,
 
             doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip],
                                       Exp, Eyp, Ezp, Bxp, Byp, Bzp,
-                                      ion_lev ? ion_lev[ip] : 0,
+                                      ion_lev ? ion_lev[ip] : 1,
                                       m, q, pusher_algo, do_crr,
 #ifdef WARPX_QED
                                       t_chi_max,
@@ -2884,7 +2900,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti,
 
                 doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip],
                                           Exp, Eyp, Ezp, Bxp, Byp, Bzp,
-                                          ion_lev ? ion_lev[ip] : 0,
+                                          ion_lev ? ion_lev[ip] : 1,
                                           m, q, pusher_algo, do_crr,
                                           t_chi_max,
                                           dt);
@@ -3124,7 +3140,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti,
         {
             doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip],
                                       Exp, Eyp, Ezp, Bxp, Byp, Bzp,
-                                      ion_lev ? ion_lev[ip] : 0,
+                                      ion_lev ? ion_lev[ip] : 1,
                                       m, q, pusher_algo, do_crr,
 #ifdef WARPX_QED
                                       t_chi_max,
@@ -3136,7 +3152,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti,
             if constexpr (qed_control == has_qed) {
                 doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip],
                                           Exp, Eyp, Ezp, Bxp, Byp, Bzp,
-                                          ion_lev ? ion_lev[ip] : 0,
+                                          ion_lev ? ion_lev[ip] : 1,
                                           m, q, pusher_algo, do_crr,
                                           t_chi_max,
                                           dt);

diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H
@@ -57,7 +57,7 @@ void doParticleMomentumPush(amrex::ParticleReal& ux,
                             const amrex::Real dt)
 {
     amrex::ParticleReal qp = a_q;
-    if (ion_lev) { qp *= ion_lev; }
+    qp *= ion_lev;
 
     if (do_crr) {
 #ifdef WARPX_QED

diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp
@@ -60,7 +60,10 @@
 #include <AMReX_ParticleUtil.H>
 #include <AMReX_Random.H>
 #include <AMReX_Utility.H>
-
+#ifdef AMREX_USE_EB
+#   include "EmbeddedBoundary/ParticleBoundaryProcess.H"
+#   include "EmbeddedBoundary/ParticleScraper.H"
+#endif
 
 #ifdef AMREX_USE_OMP
 #   include <omp.h>
@@ -291,6 +294,12 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n,
         );
     }
 
+    // Remove particles that are inside the embedded boundaries
+#ifdef AMREX_USE_EB
+    auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB();
+    scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb());
+#endif
+
     Redistribute();
 }
 

diff --git a/Source/WarpX.H b/Source/WarpX.H
@@ -132,7 +132,9 @@ public:
     MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; }
     HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; }
     MultiDiagnostics& GetMultiDiags () {return *multi_diags;}
-
+#ifdef AMREX_USE_EB
+    amrex::Vector<std::unique_ptr<amrex::MultiFab> >& GetDistanceToEB () {return m_distance_to_eb;}
+#endif
     ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; }
 
     static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom,

diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake
@@ -269,7 +269,7 @@ set(WarpX_amrex_src ""
 set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git"
     CACHE STRING
     "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)")
-set(WarpX_amrex_branch "73b215557c0e842c3e829b683939bbb7a7e12373"
+set(WarpX_amrex_branch "689144d157a0106faf3d0ae89f8d90b0250cf975"
     CACHE STRING
     "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)")
 

diff --git a/run_test.sh b/run_test.sh
@@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt
 
 # Clone AMReX and warpx-data
 git clone https://github.com/AMReX-Codes/amrex.git
-cd amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd -
+cd amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd -
 # warpx-data contains various required data sets
 git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git
 # openPMD-example-datasets contains various required data sets