diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 09f7f9e80fa..9960cdfbb29 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd - + cd ../amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Docs/source/theory/intro.rst b/Docs/source/theory/intro.rst index 10fbba679c2..2101d9d81cb 100644 --- a/Docs/source/theory/intro.rst +++ b/Docs/source/theory/intro.rst @@ -8,8 +8,7 @@ Introduction Plasma laser-driven (top) and charged-particles-driven (bottom) acceleration (rendering from 3-D Particle-In-Cell simulations). A laser beam (red and blue disks in top picture) or a charged particle beam (red dots in bottom picture) propagating (from left to right) through an under-dense plasma (not represented) displaces electrons, creating a plasma wakefield that supports very high electric fields (pale blue and yellow). These electric fields, which can be orders of magnitude larger than with conventional techniques, can be used to accelerate a short charged particle beam (white) to high-energy over a very short distance. -Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`, with -accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. +Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`. Accurate modeling of wake formation, electron self-trapping and acceleration require fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. For such simulations, the most popular algorithm is the Particle-In-Cell (or PIC) technique, diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 857f857a554..89bd5af2eab 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1575,7 +1575,7 @@ def __init__(self, name, species, scattering_processes, ndt=None, **kw): self.handle_init(kw) - def initialize_inputs(self): + def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) collision.type = 'dsmc' collision.species = [species.name for species in self.species] diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f0557e05fc4..70c62b190fb 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 73b215557c0e842c3e829b683939bbb7a7e12373 +branch = 689144d157a0106faf3d0ae89f8d90b0250cf975 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 75939c1fd12..ab11d70dfcc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 73b215557c0e842c3e829b683939bbb7a7e12373 +branch = 689144d157a0106faf3d0ae89f8d90b0250cf975 [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -3021,24 +3021,24 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py -# [Python_background_mcc_1d_tridiag] -# buildDir = . -# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -# runtime_params = -# customRunCmd = python3 PICMI_inputs_1d.py --test -# dim = 1 -# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -# target = pip_install -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 1 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# compareParticles = 0 -# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py +[Python_background_mcc_1d_tridiag] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_collisionXZ] buildDir = . @@ -3079,24 +3079,24 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py -# [Python_dsmc_1d] -# buildDir = . -# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -# runtime_params = -# customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc -# dim = 1 -# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -# target = pip_install -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 1 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# compareParticles = 0 -# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +[Python_dsmc_1d] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py [Python_ElectrostaticSphereEB] buildDir = . diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1219fe78b66..bb7ecd65daf 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -39,6 +39,10 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/ParticleBoundaryProcess.H" +# include "EmbeddedBoundary/ParticleScraper.H" +#endif #include "WarpX.H" #include @@ -1437,6 +1441,12 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + // The function that calls this is responsible for redistributing particles. } @@ -1930,6 +1940,12 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + // Redistribute the new particles that were added to the temporary container. // (This eliminates invalid particles, and makes sure that particles // are in the right tile.) @@ -2864,7 +2880,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, #ifdef WARPX_QED t_chi_max, @@ -2884,7 +2900,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, t_chi_max, dt); @@ -3124,7 +3140,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, { doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, #ifdef WARPX_QED t_chi_max, @@ -3136,7 +3152,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, if constexpr (qed_control == has_qed) { doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, t_chi_max, dt); diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index 2a6ac8a7d6c..4a82e582bfb 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -57,7 +57,7 @@ void doParticleMomentumPush(amrex::ParticleReal& ux, const amrex::Real dt) { amrex::ParticleReal qp = a_q; - if (ion_lev) { qp *= ion_lev; } + qp *= ion_lev; if (do_crr) { #ifdef WARPX_QED diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index e94aae5d573..85bb1c3f4b8 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -60,7 +60,10 @@ #include #include #include - +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/ParticleBoundaryProcess.H" +# include "EmbeddedBoundary/ParticleScraper.H" +#endif #ifdef AMREX_USE_OMP # include @@ -291,6 +294,12 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, ); } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + Redistribute(); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 5fc161451e9..81f6969ef11 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -132,7 +132,9 @@ public: MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} - +#ifdef AMREX_USE_EB + amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} +#endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 31c8919e38f..9b74c8db7fd 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "73b215557c0e842c3e829b683939bbb7a7e12373" +set(WarpX_amrex_branch "689144d157a0106faf3d0ae89f8d90b0250cf975" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 77980bb00b0..48857d264cb 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd - +cd amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets