CMake: heFFTe Support (ECP-WarpX#4986)

* CMake: heFFTe * Doc: Perlmutter heFFTe * Update Spack Dev Envs * Finalize After Testing Co-authored-by: Alfred Mishi <[email protected]> --------- Co-authored-by: Alfred Mishi <[email protected]>
RevathiJambunathan · Jun 13, 2024 · 3c4e523 · 3c4e523
1 parent 2602b54
commit 3c4e523
Show file tree

Hide file tree

Showing 14 changed files with 135 additions and 7 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -79,6 +79,7 @@ option(WarpX_LIB           "Build WarpX as a library"                   OFF)
 option(WarpX_MPI           "Multi-node support (message-passing)"       ON)
 option(WarpX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                  ON)
 option(WarpX_FFT           "FFT-based solvers"                          OFF)
+option(WarpX_HEFFTE        "Multi-node FFT-based solvers"               OFF)
 option(WarpX_PYTHON        "Python bindings"                            OFF)
 option(WarpX_SENSEI        "SENSEI in situ diagnostics"                 OFF)
 option(WarpX_QED           "QED support (requires PICSAR)"              ON)
@@ -136,6 +137,10 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE)
 
 option(WarpX_amrex_internal                    "Download & build AMReX" ON)
 
+if(WarpX_HEFFTE AND NOT WarpX_MPI)
+    message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.")
+endif()
+
 # change the default build type to Release (or RelWithDebInfo) instead of Debug
 set_default_build_type("Release")
 
@@ -174,6 +179,10 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT})
 if(WarpX_FFT)
     set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE)
 endif()
+option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE})
+if(WarpX_HEFFTE)
+    set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE)
+endif()
 
 # this defined the variable BUILD_TESTING which is ON by default
 #include(CTest)
@@ -215,6 +224,23 @@ if(WarpX_FFT)
     endif()
 endif()
 
+# multi-node FFT
+if(WarpX_HEFFTE)
+    if(WarpX_COMPUTE STREQUAL CUDA)
+        set(_heFFTe_COMPS CUDA)
+    elseif(WarpX_COMPUTE STREQUAL HIP)
+        set(_heFFTe_COMPS ROCM)
+    elseif(WarpX_COMPUTE STREQUAL SYCL)
+        set(_heFFTe_COMPS ONEAPI)
+    else()  # NOACC, OMP
+        set(_heFFTe_COMPS FFTW)  # or MKL
+    endif()
+    # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be
+    #       disabled at runtime
+
+    find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS})
+endif()
+
 # Python
 if(WarpX_PYTHON)
     find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
@@ -455,6 +481,10 @@ foreach(D IN LISTS WarpX_DIMS)
         endif()
     endif()
 
+    if(ABLASTR_HEFFTE)
+        target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte)
+    endif()
+
     if(WarpX_PYTHON)
         target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras)
         if(WarpX_PYTHON_IPO)
@@ -539,6 +569,13 @@ foreach(D IN LISTS WarpX_DIMS)
         target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT)
     endif()
 
+    if(WarpX_HEFFTE)
+        target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE)
+    endif()
+    if(ABLASTR_HEFFTE)
+        target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE)
+    endif()
+
     if(WarpX_PYTHON AND pyWarpX_VERSION_INFO)
         # for module __version__
         target_compile_definitions(pyWarpX_${SD} PRIVATE

diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst
@@ -97,6 +97,7 @@ CMake Option                  Default & Values                             Descr
 ``WarpX_PRECISION``           SINGLE/**DOUBLE**                            Floating point precision (single/double)
 ``WarpX_PARTICLE_PRECISION``  SINGLE/**DOUBLE**                            Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set
 ``WarpX_FFT``                 ON/**OFF**                                   FFT-based solvers
+``WarpX_HEFFTE``              ON/**OFF**                                   Multi-Node FFT-based solvers
 ``WarpX_PYTHON``              ON/**OFF**                                   Python bindings
 ``WarpX_QED``                 **ON**/OFF                                   QED support (requires PICSAR)
 ``WarpX_QED_TABLE_GEN``       ON/**OFF**                                   QED table generation support (requires PICSAR and Boost)
@@ -271,6 +272,7 @@ Environment Variable          Default & Values                             Descr
 ``WARPX_PRECISION``           SINGLE/**DOUBLE**                            Floating point precision (single/double)
 ``WARPX_PARTICLE_PRECISION``  SINGLE/**DOUBLE**                            Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set
 ``WARPX_FFT``                 ON/**OFF**                                   FFT-based solvers
+``WARPX_HEFFTE``              ON/**OFF**                                   Multi-Node FFT-based solvers
 ``WARPX_QED``                 **ON**/OFF                                   PICSAR QED (requires PICSAR)
 ``WARPX_QED_TABLE_GEN``       ON/**OFF**                                   QED table generation (requires PICSAR and Boost)
 ``BUILD_PARALLEL``            ``2``                                        Number of threads to use for parallel builds

diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst
@@ -23,12 +23,13 @@ Optional dependencies include:
 - for on-node accelerated compute *one of either*:
 
   - `OpenMP 3.1+ <https://www.openmp.org>`__: for threaded CPU execution or
-  - `CUDA Toolkit 11.7+ <https://developer.nvidia.com/cuda-downloads>`__: for Nvidia GPU support (see `matching host-compilers <https://gist.github.com/ax3l/9489132>`_) or
+  - `CUDA Toolkit 11.7+ <https://developer.nvidia.com/cuda-downloads>`__: for Nvidia GPU support (see `matching host-compilers <https://gist.github.com/ax3l/9489132>`__) or
   - `ROCm 5.2+ (5.5+ recommended) <https://gpuopen.com/learn/amd-lab-notes/amd-lab-notes-rocm-installation-readme/>`__: for AMD GPU support
-- `FFTW3 <http://www.fftw.org>`_: for spectral solver (PSATD) support when running on CPU or SYCL
+- `FFTW3 <http://www.fftw.org>`__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL
 
   - also needs the ``pkg-config`` tool on Unix
-- `BLAS++ <https://github.com/icl-utk-edu/blaspp>`_ and `LAPACK++ <https://github.com/icl-utk-edu/lapackpp>`_: for spectral solver (PSATD) support in RZ geometry
+- `heFFTe 2.4.0+ <https://github.com/icl-utk-edu/heffte`__: for multi-node spectral solver (IGF) support
+- `BLAS++ <https://github.com/icl-utk-edu/blaspp>`__ and `LAPACK++ <https://github.com/icl-utk-edu/lapackpp>`__: for spectral solver (PSATD) support in RZ geometry
 - `Boost 1.66.0+ <https://www.boost.org/>`__: for QED lookup tables generation support
 - `openPMD-api 0.15.1+ <https://github.com/openPMD/openPMD-api>`__: we automatically download and compile a copy of openPMD-api for openPMD I/O support
 

diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst
@@ -153,7 +153,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/warpx
          rm -rf build_pm_gpu
 
-         cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_gpu -j 16
 
       The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``.
@@ -164,7 +164,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/warpx
          rm -rf build_pm_gpu_py
 
-         cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_gpu_py -j 16 --target pip_install
 
    .. tab-item:: CPU Nodes
@@ -174,7 +174,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/warpx
          rm -rf build_pm_cpu
 
-         cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_cpu -j 16
 
       The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``.
@@ -184,7 +184,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
          rm -rf build_pm_cpu_py
 
-         cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
          cmake --build build_pm_cpu_py -j 16 --target pip_install
 
 Now, you can :ref:`submit Perlmutter compute jobs <running-cpp-perlmutter>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).

diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml
@@ -23,6 +23,7 @@ spack:
   - conduit ~fortran
   - fftw
   - hdf5 ~fortran
+  - heffte ~cuda +fftw
   - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm
   - mpi
   - llvm-openmp

diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml
@@ -25,6 +25,7 @@ spack:
   - cuda
   - fftw
   - hdf5
+  - heffte
   - lapackpp
   - mpi
   - pkgconfig

diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml
@@ -22,6 +22,7 @@ spack:
   - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei
   - fftw
   - hdf5
+  - heffte ~cuda +fftw
   - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm
   - mpi
   - pkgconfig

diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml
@@ -21,6 +21,7 @@ spack:
   - cmake
   - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei
   - hdf5
+  - heffte
   - hip
   - lapackpp
   - llvm-amdgpu

diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh
@@ -109,6 +109,45 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B
 cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16
 rm -rf ${build_dir}/lapackpp-pm-cpu-build
 
+# heFFTe
+if [ -d $HOME/src/heffte ]
+then
+  cd $HOME/src/heffte
+  git fetch --prune
+  git checkout v2.4.0
+  git pull
+  cd -
+else
+  git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte
+fi
+rm -rf ${HOME}/src/heffte-pm-cpu-build
+cmake \
+    -S ${HOME}/src/heffte               \
+    -B ${build_dir}/heffte-pm-cpu-build \
+    -DBUILD_SHARED_LIBS=ON              \
+    -DCMAKE_BUILD_TYPE=Release          \
+    -DCMAKE_CXX_STANDARD=17             \
+    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON  \
+    -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0  \
+    -DHeffte_DISABLE_GPU_AWARE_MPI=ON   \
+    -DHeffte_ENABLE_AVX=ON              \
+    -DHeffte_ENABLE_AVX512=OFF          \
+    -DHeffte_ENABLE_FFTW=ON             \
+    -DHeffte_ENABLE_CUDA=OFF            \
+    -DHeffte_ENABLE_ROCM=OFF            \
+    -DHeffte_ENABLE_ONEAPI=OFF          \
+    -DHeffte_ENABLE_MKL=OFF             \
+    -DHeffte_ENABLE_DOXYGEN=OFF         \
+    -DHeffte_SEQUENTIAL_TESTING=OFF     \
+    -DHeffte_ENABLE_TESTING=OFF         \
+    -DHeffte_ENABLE_TRACING=OFF         \
+    -DHeffte_ENABLE_PYTHON=OFF          \
+    -DHeffte_ENABLE_FORTRAN=OFF         \
+    -DHeffte_ENABLE_SWIG=OFF            \
+    -DHeffte_ENABLE_MAGMA=OFF
+cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16
+rm -rf ${build_dir}/heffte-pm-cpu-build
+
 
 # Python ######################################################################
 #

diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh
@@ -109,6 +109,44 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B
 cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16
 rm -rf ${build_dir}/lapackpp-pm-gpu-build
 
+# heFFTe
+if [ -d $HOME/src/heffte ]
+then
+  cd $HOME/src/heffte
+  git fetch --prune
+  git checkout v2.4.0
+  cd -
+else
+  git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte
+fi
+rm -rf ${HOME}/src/heffte-pm-gpu-build
+cmake \
+    -S ${HOME}/src/heffte               \
+    -B ${build_dir}/heffte-pm-gpu-build \
+    -DBUILD_SHARED_LIBS=ON              \
+    -DCMAKE_BUILD_TYPE=Release          \
+    -DCMAKE_CXX_STANDARD=17             \
+    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON  \
+    -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0  \
+    -DHeffte_DISABLE_GPU_AWARE_MPI=OFF  \
+    -DHeffte_ENABLE_AVX=OFF             \
+    -DHeffte_ENABLE_AVX512=OFF          \
+    -DHeffte_ENABLE_FFTW=OFF            \
+    -DHeffte_ENABLE_CUDA=ON             \
+    -DHeffte_ENABLE_ROCM=OFF            \
+    -DHeffte_ENABLE_ONEAPI=OFF          \
+    -DHeffte_ENABLE_MKL=OFF             \
+    -DHeffte_ENABLE_DOXYGEN=OFF         \
+    -DHeffte_SEQUENTIAL_TESTING=OFF     \
+    -DHeffte_ENABLE_TESTING=OFF         \
+    -DHeffte_ENABLE_TRACING=OFF         \
+    -DHeffte_ENABLE_PYTHON=OFF          \
+    -DHeffte_ENABLE_FORTRAN=OFF         \
+    -DHeffte_ENABLE_SWIG=OFF            \
+    -DHeffte_ENABLE_MAGMA=OFF
+cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16
+rm -rf ${build_dir}/heffte-pm-gpu-build
+
 
 # Python ######################################################################
 #

diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example
@@ -19,11 +19,13 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1
 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH
 
 export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH
 
 export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH}
 

diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example
@@ -23,11 +23,13 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.2
 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH
 
 export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH
 
 export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH}
 

diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake
@@ -453,6 +453,7 @@ function(warpx_print_summary)
     message("    PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}")
     message("    PRECISION: ${WarpX_PRECISION}")
     message("    FFT Solvers: ${WarpX_FFT}")
+    message("    heFFTe: ${WarpX_HEFFTE}")
     message("    PYTHON: ${WarpX_PYTHON}")
     if(WarpX_PYTHON)
         message("    PYTHON IPO: ${WarpX_PYTHON_IPO}")

diff --git a/setup.py b/setup.py
@@ -97,6 +97,7 @@ def build_extension(self, ext):
             '-DWarpX_PRECISION=' + WARPX_PRECISION,
             '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION,
             '-DWarpX_FFT:BOOL=' + WARPX_FFT,
+            '-DWarpX_HEFFTE:BOOL=' + WARPX_HEFFTE,
             '-DWarpX_PYTHON:BOOL=ON',
             '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO,
             '-DWarpX_QED:BOOL=' + WARPX_QED,
@@ -206,6 +207,7 @@ def build_extension(self, ext):
 WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE')
 WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION)
 WARPX_FFT = env.pop('WARPX_FFT', 'OFF')
+WARPX_HEFFTE = env.pop('WARPX_HEFFTE', 'OFF')
 WARPX_QED = env.pop('WARPX_QED', 'ON')
 WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF')
 WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3')
-Original file line number
+Diff line change
@@ Expand Up / @@ -25,6 +25,7 @@ spack: @@
       - cuda
       - fftw
       - hdf5
+      - heffte
       - lapackpp
       - mpi
       - pkgconfig
@@ Expand Down @@