Merge branch 'development' into 2d_spherical_rotation

.github/workflows/clang-tidy.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          .github/workflows/dependencies_clang-tidy-apt-llvm.sh 19
+          .github/workflows/dependencies/dependencies_clang-tidy-apt-llvm.sh 19
 
       - name: Compile flame_wave
         run: |

.github/workflows/dependencies_hip.sh → .github/workflows/dependencies/dependencies_hip.sh

@@ -40,6 +40,7 @@ echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/r
 
 # we should not need to export HIP_PATH=/opt/rocm/hip with those installs
 
+sudo apt-get clean
 sudo apt-get update
 
 # Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#installing-development-packages-for-cross-compilation
@@ -56,7 +57,9 @@ sudo apt-get install -y --no-install-recommends \
     roctracer-dev   \
     rocprofiler-dev \
     rocrand-dev     \
-    rocprim-dev
+    rocfft-dev      \
+    rocprim-dev     \
+    rocsparse-dev
 
 # hiprand-dev is a new package that does not exist in old versions
 sudo apt-get install -y --no-install-recommends hiprand-dev || true

.github/workflows/dependencies/dependencies_nvcc.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Copyright 2020-2022 Axel Huebl
+#
+# License: BSD-3-Clause-LBNL
+
+set -eu -o pipefail
+
+# `man apt.conf`:
+#   Number of retries to perform. If this is non-zero APT will retry
+#   failed files the given number of times.
+echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
+
+sudo apt-get -qqq update
+sudo apt-get install -y \
+    build-essential     \
+    ca-certificates     \
+    cmake               \
+    g++                 \
+    gfortran            \
+    gnupg               \
+    libopenmpi-dev      \
+    openmpi-bin         \
+    pkg-config          \
+    wget
+
+VERSION_DOTTED=${1-12.0} && VERSION_DASHED=$(sed 's/\./-/' <<< $VERSION_DOTTED)
+curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb
+sudo dpkg -i cuda-keyring_1.0-1_all.deb
+sudo apt-get update
+sudo apt-get install -y \
+    cuda-command-line-tools-$VERSION_DASHED \
+    cuda-compiler-$VERSION_DASHED           \
+    cuda-cupti-dev-$VERSION_DASHED          \
+    cuda-minimal-build-$VERSION_DASHED      \
+    cuda-nvml-dev-$VERSION_DASHED           \
+    cuda-nvtx-$VERSION_DASHED               \
+    libcublas-dev-$VERSION_DASHED           \
+    libcufft-dev-$VERSION_DASHED            \
+    libcurand-dev-$VERSION_DASHED           \
+    libcusparse-dev-$VERSION_DASHED
+sudo ln -s cuda-$VERSION_DOTTED /usr/local/cuda

.github/workflows/gpu_action.yml

@@ -29,14 +29,7 @@ jobs:
           sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0
 
       - name: Install CUDA
-        # from https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04&target_type=deb_network
-        run: |
-          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
-          sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600
-          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
-          sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
-          sudo apt-get update
-          sudo apt-get -y install cuda-toolkit-11-8
+        run: .github/workflows/dependencies/dependencies_nvcc.sh 11.8
 
       - name: Install hypre
         run: |

.github/workflows/hip.yml

@@ -24,7 +24,7 @@ jobs:
           cd ../..
 
       - name: Dependencies
-        run: .github/workflows/dependencies_hip.sh
+        run: .github/workflows/dependencies/dependencies_hip.sh
 
       - name: compile flame_wave
         run: |

Docs/source/diffusion.rst

@@ -4,7 +4,7 @@
 Thermal Diffusion
 *****************
 
-Castro incorporates explicit thermal diffusion into the energy equation.
+Castro incorporates explicit thermal diffusion into the energy equations.
 In terms of the specific internal energy, :math:`e`, this appears as:
 
 .. math:: \rho \frac{De}{Dt} + p \nabla \cdot \ub = \nabla \cdot \kth \nabla T
@@ -20,15 +20,82 @@ where :math:`\kth` is the thermal conductivity, with units
 
      USE_DIFFUSION=TRUE
 
-It is treated explicitly, by constructing the contribution to the evolution as a
-source term. This is time-centered to achieve second-order accuracy
+Thermal Diffusion related source codes are contained in the ``diffusion`` directory.
+Thermal Diffusion is treated explicitly, by constructing the contribution to the
+evolution as a source term. This is time-centered to achieve second-order accuracy
 in time.
 
+Overall Procedure
+=================
+
+Computing Thermal Conductivity
+------------------------------
+The main function that computes the diffusion term is ``getTempDiffusionTerm()``.
+Within ``getTempDiffusionTerm()``, it first calculates the cell centered
+thermal conductivity, :math:`\kth` contained in the variable ``coeff_cc``
+using the function ``fill_temp_cond()`` located in ``diffusion_util.cpp``.
+``fill_temp_cond()`` fills an ``eos_state`` using the
+input conserved variables, which is used to calculate :math:`\kth` via
+``conductivity(eos_state)``. ``conductivity()`` routine is supplied via
+the ``Microphysics`` package. See :ref:`sec:conductivities` to see the
+specific choices of conductivity routines available.
+
+.. note::
+   The diffusion approximation breaks down at the surface of stars,
+   where the density rapidly drops and the mean free path becomes
+   large. In those instances, you should use the flux limited diffusion
+   module in Castro to evolve a radiation field.
+
+Now :math:`\kth` is reset to 0 unless
+:math:`\rho \gt \mathrm{castro::diffuse\_cutoff\_density}`.
+And if :math:`\rho \lt \mathrm{castro::diffuse\_cutoff\_density\_hi}`,
+a linear scaling of :math:`\kth` is done as:
+
+.. math::
+
+   \kth = \kth \cdot \frac{\rho - \mathtt{castro.diffuse\_cutoff\_density}}{\mathtt{castro.diffuse\_cutoff\_density\_hi} - \mathtt{castro.diffuse\_cutoff\_density}}
+
+Lastly, :math:`\kth` is scaled with ``castro::diffuse_cond_scale_fac``,
+a runtime parameter controlled by the user.
+
+After obtaining cell-centered :math:`\kth`, we do an average along
+i, j, and k depending on the direction to obtain face-centered MultiFabs.
+This is stored in ``coeffs``, a vector of MultiFabs, and the number of
+MultiFabs corresponds to geometry dimension, since a :math:`\nabla` operator
+will be applied to it later.
+These Multifabs have 1 ghost cells due to the nature of MLMG solvers.
+
+.. _sec:thermal_diffusion:
+
+Computing Thermal Diffusion
+---------------------------
+We are now ready to compute :math:`\nabla \cdot \kth \nabla T`
+after obtaining :math:`\kth`. This is done in the ``applyop_mlmg()`` function
+in ``Diffusion.cpp``. It defines ``mlabec`` an instance of class
+``MLABecLaplacian`` which defines the Laplacian of the form:
+
+.. math::
+   (A\alpha - B\nabla \cdot \beta \nabla) \phi = f
+
+where A and B are constant scalars, and :math:`\alpha` and :math:`\beta`
+are scalar fields. In order to make it correctly represents our diffusion term,
+we set A = 0 and B = -1, which is done via ``mlabec.setScalars(0.0, -1.0)``.
+Now we recognize :math:`\beta = \kth`, which needs to be an array of MultiFab,
+corresponding to dimension. This is done via ``mlabec.setBCoeffs()``.
+
+One of the important flag that we need to pass in is to set ``setMetricTerm(true)``.
+This enables modifications due to curvilinear coordinates.
+
+Finally we create an instance of ``MLMG`` using ``mlabec``, and call
+``mlmg.apply()``, which simply evaluates the LHS but do not solve it.
+See more information in the amrex documentation:
+https://amrex-codes.github.io/amrex/docs_html/LinearSolvers.html
+
 
 Timestep Limiter
 ================
 
-Castro integrates diffusion explicitly in time—this means that
+Castro integrates diffusion explicitly in time; this means that
 there is a diffusion timestep limiter.
 
 To see the similarity to the thermal diffusion equation, consider the
@@ -59,27 +126,16 @@ The following parameter affects diffusion:
 
       castro.diffuse_temp = 1
       castro.do_hydro = 0
+* ``castro.diffuse_cond_scale_fac``: a linear scaling to :math:`\kth`. (default 0).
 
-.. index:: castro.diffusion_cutoff_density, castro.diffusion_cutoff_density_hi
-
-The diffusion approximation breaks down at the surface of stars,
-where the density rapidly drops and the mean free path becomes
-large. In those instances, you should use the flux limited diffusion
-module in Castro to evolve a radiation field. However, if your
-interest is only on the diffusion in the interior, you can use
-the parameters:
+* ``castro.diffuse_cutoff_density``: density under which :math:`\kth` is set to 0.
+  (Default: -1e200)
 
- * ``castro.diffuse_cutoff_density``
+* ``castro.diffuse_cutoff_density_hi``: density under which a linear scaling is
+  applied to :math:`\kth`, see section :ref:`sec:thermal_diffusion` for details.
+  (Default: -1e200)
 
- * ``castro.diffuse_cutoff_density_hi``
-
-to specify a density,
-below which, diffusion is not modeled. This is implemented in the
-code by linearly scaling the conductivity to zero between these limits, e.g.,
-
-.. math::
-
-   \kth = \kth \cdot \frac{\rho - \mathtt{castro.diffuse\_cutoff\_density}}{\mathtt{castro.diffuse\_cutoff\_density\_hi} - \mathtt{castro.diffuse\_cutoff\_density}}
+.. _sec:conductivities:
 
 Conductivities
 ==============

Docs/source/radiation.rst

@@ -126,7 +126,7 @@ frequency, and :math:`m`, :math:`n` and :math:`p` are constants. For the gray so
 
 ::
 
-    Opacity_dir := rad_power_law
+    OPACITY_DIR := rad_power_law
 
 in your GNUmakefile. See § \ `3.3.1 <#sec:opacpars>`__ for instructions on how
 to configure the parameters used for this opacity setup. If you would prefer a different

Exec/radiation_tests/Rad2Tshock/GNUmakefile

@@ -17,7 +17,7 @@ EOS_DIR     := gamma_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/radiation_tests/RadBlastWave/GNUmakefile

@@ -15,7 +15,7 @@ EOS_DIR     := gamma_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/radiation_tests/RadBreakout/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 EOS_DIR     = breakout
 NETWORK_DIR := general_null
 NETWORK_INPUTS = breakout.net
-Opacity_dir = breakout
+OPACITY_DIR = breakout
 
 PROBLEM_DIR ?= ./
 

Exec/radiation_tests/RadFront/GNUmakefile

@@ -17,7 +17,7 @@ EOS_DIR     := gamma_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/radiation_tests/RadShestakovBolstad/GNUmakefile

@@ -17,7 +17,7 @@ EOS_DIR     := rad_power_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 # number of radiation groups -  this needs to correspond to radiation.nGroups in
 # the inputs file

Exec/radiation_tests/RadSourceTest/GNUmakefile

@@ -20,7 +20,7 @@ EOS_DIR     := rad_power_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 # use radiation interpbndry
 RAD_INTERP = TRUE

Exec/radiation_tests/RadSphere/GNUmakefile

@@ -17,7 +17,7 @@ EOS_DIR     := rad_power_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 # number of radiation groups -  this needs to correspond to radiation.nGroups in
 # the inputs file

Exec/radiation_tests/RadSuOlson/GNUmakefile

@@ -16,7 +16,7 @@ EOS_DIR     := rad_power_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/radiation_tests/RadSuOlsonMG/GNUmakefile

@@ -17,7 +17,7 @@ EOS_DIR     := rad_power_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 NGROUPS := 2
 

Exec/radiation_tests/RadThermalWave/GNUmakefile

@@ -17,7 +17,7 @@ EOS_DIR     := gamma_law
 NETWORK_DIR := general_null
 NETWORK_INPUTS := gammalaw.net
 
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/science/bwp-rad/GNUmakefile

@@ -29,7 +29,7 @@ NETWORK_DIR := general_null
 NETWORK_INPUTS := H_He.net
 
 # power-law opacity
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/science/planet/GNUmakefile

@@ -33,7 +33,7 @@ NETWORK_DIR := general_null
 NETWORK_INPUTS := ./taeho.net
 
 # power-law opacity
-Opacity_dir := rad_power_law
+OPACITY_DIR := rad_power_law
 
 PROBLEM_DIR ?= ./
 

Exec/unit_tests/diffusion_test/GNUmakefile

@@ -5,7 +5,7 @@ DEBUG      = FALSE
 
 DIM        = 2
 
-COMP	   = gnu
+COMP       = gnu
 
 USE_MPI    = TRUE
 USE_OMP    = FALSE