Merge branch 'main' of github.com:AMReX-FHD/FHDeX into main

exec/DSMC/GNUmakefile

@@ -11,6 +11,7 @@ USE_OMP   = FALSE
 USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
+MAX_SPEC      = 8
 
 TINY_PROFILE  = FALSE
 USE_PARTICLES = TRUE
@@ -84,3 +85,6 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)

exec/DSMC_granular/GNUmakefile

@@ -11,6 +11,7 @@ USE_OMP   = FALSE
 USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
+MAX_SPEC      = 8
 
 TINY_PROFILE  = FALSE
 USE_PARTICLES = TRUE
@@ -90,3 +91,7 @@ include $(AMREX_HOME)/Tools/GNUMake/Make.rules
 ifeq ($(findstring cgpu, $(HOST)), cgpu)
 CXXFLAGS += $(FFTW)
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/Ek_calculator/GNUmakefile

@@ -8,6 +8,7 @@ USE_OMP   = FALSE
 USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
+MAX_SPEC      = 8
 
 TINY_PROFILE = FALSE
 
@@ -38,3 +39,7 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/chemistry_testing/GNUmakefile

@@ -10,6 +10,7 @@ USE_OMP      = FALSE
 USE_CUDA     = FALSE
 COMP         = gnu
 DIM          = 3
+MAX_SPEC      = 8
 
 TINY_PROFILE = FALSE
 
@@ -35,3 +36,7 @@ INCLUDE_LOCATIONS += ../../src_common/
 include $(AMREX_HOME)/Src/Base/Make.package
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/compressible/GNUmakefile

@@ -9,6 +9,7 @@ USE_CUDA      = FALSE
 COMP          = gnu
 DIM           = 3
 TINY_PROFILE  = FALSE
+MAX_SPEC      = 8
 
 USE_PARTICLES = FALSE
 
@@ -55,3 +56,7 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/compressible_stag/GNUmakefile

@@ -6,11 +6,14 @@ DEBUG         = FALSE
 USE_MPI       = TRUE
 USE_OMP       = FALSE
 USE_CUDA      = TRUE
+USE_HIP       = FALSE
 COMP          = gnu
 DIM           = 3
 TINY_PROFILE  = FALSE
+MAX_SPEC      = 8
 
 USE_PARTICLES = FALSE
+DO_TURB 	  = FALSE
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
@@ -56,6 +59,18 @@ endif
 
 ifeq ($(USE_CUDA),TRUE)
   LIBRARIES += -lcufft
+else ifeq ($(USE_HIP),TRUE)
+  # Use rocFFT.  ROC_PATH is defined in amrex
+  INCLUDE_LOCATIONS += $(ROC_PATH)/rocfft/include
+  LIBRARY_LOCATIONS += $(ROC_PATH)/rocfft/lib
+  LIBRARIES += -L$(ROC_PATH)/rocfft/lib -lrocfft
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+ifeq ($(DO_TURB), TRUE)
+  DEFINES += -DTURB
+endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)

exec/compressible_stag/build_frontier.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/bash
+
+## load necessary modules 
+module load cmake/3.23.2
+module load craype-accel-amd-gfx90a
+module load rocm/5.2.0  # waiting for 5.6 for next bump
+module load cray-mpich
+module load cce/15.0.0  # must be loaded after rocm
+
+# GPU-aware MPI
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# optimize CUDA compilation for MI250X
+export AMREX_AMD_ARCH=gfx90a
+
+# compiler environment hints
+export CC=$(which hipcc)
+export CXX=$(which hipcc)
+export FC=$(which ftn)
+export CFLAGS="-I${ROCM_PATH}/include"
+export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
+export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"
+
+make -j18 USE_CUDA=FALSE USE_HIP=TRUE DO_TURB=TRUE

exec/compressible_stag/inputs_taylor_green_turbulence

@@ -0,0 +1,97 @@
+  # Problem specification
+  # mean free path (Ar): lambda = 6.26e-6 cm
+  # dx = dy = dz = 2*lambda = 1.252e-5
+  prob_lo = 0.0 0.0 0.0      # physical lo coordinate
+  prob_hi = 0.080128 0.080128 0.080128 # physical hi coordinate
+
+  # Number of ghost cells, conserved, and primitive variables
+  # ---------------------
+  ngc = 2 2 2
+  nspecies = 2
+  nvars = 7
+  nprimvars = 10
+
+  # number of cells in domain
+  n_cells = 128 128 128
+  # max number of cells in a box
+  max_grid_size = 128 128 128
+
+  # Time-step control
+  fixed_dt = 5e-9
+  stoch_stress_form = 1
+
+  # Controls for number of steps between actions
+  max_step = 1000000
+  plot_int = 5000
+
+  # Multispecies toggle
+  # if algorithm_type = 1, single component
+  # if algorithm_type = 2, multispecies
+  algorithm_type = 2
+
+  # Viscous tensor form
+  # if visc_type = 1, L = not-symmetric (bulk viscosity = 0)
+  # if visc_type = 2, L = symmetric (bulk viscosity = 0)
+  # if visc_type = 3, L = symmetric + bulk viscosity
+  visc_type = 2
+
+  # Advection method
+  # if advection_type = 1, interpolate primitive quantities
+  # if advection_type = 2, interpolate conserved quantities
+  advection_type = 1
+
+  # Problem specification
+  # if prob_type = 1, constant species concentration
+  # if prob_type = 2, Rayleigh-Taylor instability
+  # if prob_type = 3, diffusion barrier
+  prob_type = 4
+
+  # Initial parameters
+  k_B = 1.38064852e-16 # [units: cm2*g*s-2*K-1]
+  Runiv = 8.314462175e7
+  T_init = 273
+  rho0 = 0.0017775409151219938
+  rhobar = 0.5 0.5
+  mach0 = 0.5
+  turbForcing = 1
+
+  struct_fact_int = -10
+  n_steps_skip = 0
+
+  # Boundary conditions:
+  # NOTE: setting bc_vel to periodic sets all the other bc's to periodic)
+  # bc_vel:   -1 = periodic  
+  #            1 = slip
+  #            2 = no-slip
+  # bc_mass:  -1 = periodic
+  #            1 = wall
+  #            2 = reservoir (set bc_Yk or bc_Xk in compressible namelist)
+  # bc_therm: -1 = periodic
+  #            1 = adiabatic
+  #            2 = isothermal (set with t_lo/hi in common namelist)
+  bc_vel_lo = -1 -1 -1
+  bc_vel_hi = -1 -1 -1
+  bc_mass_lo  = -1 -1 -1
+  bc_mass_hi  = -1 -1 -1
+  bc_therm_lo = -1 -1 -1
+  bc_therm_hi = -1 -1 -1
+
+  # Temperature if thermal BC specified
+  t_lo = 273 273 273
+  t_hi = 273 273 273
+
+  #Kinetic species info
+  #--------------
+  molmass  = 39.9480 39.9480 
+  diameter = 3.66e-8 3.66e-8
+
+  # Enter negative dof to use hcv & hcp values
+  dof =  3 3
+  hcv = -1 -1
+  hcp = -1 -1
+
+  # write out means and variances to plotfile
+  transport_type = 1
+  plot_means = 0
+  plot_vars = 0
+  plot_covars = 0

exec/compressible_stag_mui/GNUmakefile

@@ -14,6 +14,7 @@ TINY_PROFILE  = FALSE
 USE_MUI       = TRUE
 USE_SLURM     = FALSE
 USE_AMREX_MPMD = TRUE
+MAX_SPEC      = 8
 
 USE_PARTICLES = FALSE
 
@@ -122,3 +123,7 @@ endif
 ifeq ($(USE_SLURM), TRUE)
   DEFINES += -DSLURM
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/dean_kow/GNUmakefile

@@ -6,6 +6,7 @@ USE_MPI   = TRUE
 USE_OMP   = FALSE
 COMP      = gcc
 DIM       = 2
+MAX_SPEC      = 8
 
 USE_CUDA  = FALSE
 USE_HIP   = FALSE
@@ -33,3 +34,6 @@ INCLUDE_LOCATIONS += ../../src_common/
 include $(AMREX_HOME)/Src/Base/Make.package
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)

exec/hydro/GNUmakefile

@@ -8,6 +8,7 @@ USE_OMP   = FALSE
 USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
+MAX_SPEC      = 8
 
 TINY_PROFILE = FALSE
 
@@ -52,3 +53,7 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/immersedIons/GNUmakefile

@@ -11,6 +11,7 @@ USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
 DSMC      = FALSE
+MAX_SPEC      = 8
 
 TINY_PROFILE  = FALSE
 USE_PARTICLES = TRUE
@@ -85,3 +86,6 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)

exec/immersed_boundary/channel_dumbbell/GNUmakefile

@@ -17,6 +17,7 @@ USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
 DSMC      = TRUE
+MAX_SPEC      = 8
 
 TINY_PROFILE = TRUE
 USE_PARTICLES = TRUE
@@ -77,3 +78,6 @@ endif
 # else
 #   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 # endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)

exec/immersed_boundary/channel_multiblob/GNUmakefile

@@ -17,6 +17,7 @@ USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
 DSMC      = TRUE
+MAX_SPEC      = 8
 
 TINY_PROFILE = FALSE
 USE_PARTICLES = TRUE
@@ -81,3 +82,7 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/immersed_boundary/channel_rigid/GNUmakefile

@@ -17,6 +17,7 @@ USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
 DSMC      = TRUE
+MAX_SPEC      = 8
 
 TINY_PROFILE = FALSE
 USE_PARTICLES = TRUE
@@ -76,3 +77,7 @@ ifeq ($(USE_CUDA),TRUE)
 else
   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/immersed_boundary/flagellum/GNUmakefile

@@ -17,6 +17,7 @@ USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
 DSMC      = TRUE
+MAX_SPEC      = 8
 
 TINY_PROFILE = TRUE
 USE_PARTICLES = TRUE
@@ -77,3 +78,7 @@ endif
 # else
 #   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 # endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)
+

exec/immersed_boundary/taylor_line/GNUmakefile

@@ -17,6 +17,7 @@ USE_CUDA  = FALSE
 COMP      = gnu
 DIM       = 3
 DSMC      = TRUE
+MAX_SPEC      = 8
 
 TINY_PROFILE = TRUE
 USE_PARTICLES = TRUE
@@ -77,3 +78,6 @@ endif
 # else
 #   LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
 # endif
+
+MAXSPECIES := $(strip $(MAX_SPEC))
+DEFINES += -DMAX_SPECIES=$(MAXSPECIES)