Merge branch 'development' into shock_paper_detonation

.editorconfig

@@ -0,0 +1,49 @@
+# http://EditorConfig.org
+#
+# precedence of rules is bottom to top
+
+# this is the top-most EditorConfig file
+root = true
+
+
+[*.{c,h,cpp,hpp,H,py}]
+# 4 space indentation
+indent_style = space
+indent_size = 4
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+# unix-style newlines
+end_of_line = lf
+
+# newline ending in files
+insert_final_newline = true
+
+
+[*.md]
+# two end of line whitespaces are newlines without a paragraph
+trim_trailing_whitespace = false
+
+
+[*.rst]
+# Enforce UTF-8 encoding
+charset = utf-8
+
+# Unix-style newlines
+end_of_line = lf
+
+# Newline ending in files
+insert_final_newline = true
+
+# 3 space indentation
+indent_style = space
+indent_size = 3
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+[{Makefile,GNUmakefile,Make.*}]
+# TABs are part of its syntax
+indent_style = tab
+indent_size = unset

.zenodo.json

@@ -17,6 +17,11 @@
             "name": "Bell, John",
             "orcid": "0000-0002-5749-334X"
         },
+        {
+            "affiliation": "Department of Physics and Astronomy, Stony Brook University",
+            "name": "Chen, Zhi",
+            "orcid": "0000-0002-2839-107X"
+        },
         {
             "name": "Harpole, Alice",
             "orcid": "0000-0002-1530-781X"

Docs/source/gpu.rst

@@ -0,0 +1,58 @@
+*********************
+GPU Programming Model
+*********************
+
+CPUs and GPUs have separate memory, which means that working on both
+the host and device may involve managing the transfer of data between
+the memory on the host and that on the GPU.
+
+In Castro, the core design when running on GPUs is that all of the compute
+should be done on the GPU.
+
+When we compile with ``USE_CUDA=TRUE`` or ``USE_HIP=TRUE``, AMReX will allocate
+a pool of memory on the GPUs and all of the ``StateData`` will be stored there.
+As long as we then do all of the computation on the GPUs, then we don't need
+to manage any of the data movement manually.
+
+.. note::
+
+   We can tell AMReX to allocate the data using managed-memory by
+   setting:
+
+   ::
+
+      amrex.the_arena_is_managed = 1
+
+   This is generally not needed.
+
+The programming model used throughout Castro is C++-lambda-capturing
+by value.  We access the ``FArrayBox`` stored in the ``StateData``
+``MultiFab`` by creating an ``Array4`` object.  The ``Array4`` does
+not directly store a copy of the data, but instead has a pointer to
+the data in the ``FArrayBox``.  When we capture the ``Array4`` by
+value in the GPU kernel, the GPU gets access to the pointer to the
+underlying data.
+
+
+Most AMReX functions will work on the data directly on the GPU (like
+``.setVal()``).
+
+In rare instances where we might need to operate on the data on the
+host, we can force a copy to the host, do the work, and then copy
+back.  For an example, see the reduction done in  ``Gravity.cpp``.
+
+.. note::
+
+   For a thorough discussion of how the AMReX GPU offloading works
+   see :cite:`amrex-ecp`.
+
+
+Runtime parameters
+------------------
+
+The main exception for all data being on the GPUs all the time are the
+runtime parameters.  At the moment, these are allocated as managed
+memory and stored in global memory.  This is simply to make it easier
+to read them in and initialize them on the CPU at runtime.
+
+

Docs/source/index.rst

@@ -21,6 +21,7 @@ https://github.com/amrex-astro/Castro
    mpi_plus_x
    FlowChart
    software
+   gpu
    problem_setups
    timestepping
    creating_a_problem

Docs/source/mpi_plus_x.rst

@@ -106,6 +106,19 @@ To enable this, compile with::
 
    to the ``make`` line or ``GNUmakefile``.
 
+.. note::
+
+   CUDA 11.2 and later can do link time optimization.  This can
+   increase performance by 10-30% (depending on the application), but
+   may greatly increase the compilation time.  This is disabled by
+   default.  To enable link time optimization, add:
+
+   .. code::
+
+      CUDA_LTO=TRUE
+
+   to the ``make`` line of ``GNUmakefile``.
+
 AMD GPUs
 --------
 
@@ -123,4 +136,3 @@ Working at Supercomputing Centers
 Our best practices for running any of the AMReX Astrophysics codes
 at different supercomputing centers is produced in our workflow
 documentation: https://amrex-astro.github.io/workflow/
-

Docs/source/refs.bib

@@ -1131,3 +1131,22 @@ @ARTICLE{doubledet2024
 title = {Sensitivity of Simulations of Double-detonation Type Ia Supernovae to Integration Methodology},
 journal = {The Astrophysical Journal},
 }
+
+
+@ARTICLE{amrex-ecp,
+       author = {{Myers}, Andrew and {Zhang}, Weiqun and {Almgren}, Ann and {Antoun}, Thierry and {Bell}, John and {Huebl}, Axel and {Sinn}, Alexander},
+        title = "{AMReX and pyAMReX: Looking Beyond ECP}",
+      journal = {arXiv e-prints},
+     keywords = {Computer Science - Distributed, Parallel, and Cluster Computing},
+         year = 2024,
+        month = mar,
+          eid = {arXiv:2403.12179},
+        pages = {arXiv:2403.12179},
+          doi = {10.48550/arXiv.2403.12179},
+archivePrefix = {arXiv},
+       eprint = {2403.12179},
+ primaryClass = {cs.DC},
+       adsurl = {https://ui.adsabs.harvard.edu/abs/2024arXiv240312179M},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+

Exec/gravity_tests/DustCollapse/GNUmakefile

@@ -22,7 +22,7 @@ CASTRO_HOME  ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/EOS
 EOS_DIR     := gamma_law
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/Networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/Networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/gravity_tests/hse_convergence/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS := triple_alpha_plus_o.net
 

Exec/gravity_tests/hydrostatic_adjust/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/Sod_stellar/GNUmakefile

@@ -21,7 +21,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/gamma_law_bubble/Problem_Derive.cpp

@@ -52,11 +52,11 @@ void ca_derpi(const Box& bx, FArrayBox& derfab, int dcomp, int /*ncomp*/,
     eos_state.T = dat(i,j,k,UTEMP);
     eos_state.e = dat(i,j,k,UEINT) * rhoInv;
     for (int n = 0; n < NumSpec; n++) {
-      eos_state.xn[n] = dat(i,j,k,UFS+n) / dat(i,j,k,URHO);
+      eos_state.xn[n] = dat(i,j,k,UFS+n) * rhoInv;
     }
 #if NAUX_NET > 0
     for (int n = 0; n < NumAux; n++) {
-      eos_state.aux[n] = dat(i,j,k,UFX+n) / dat(i,j,k,URHO);
+      eos_state.aux[n] = dat(i,j,k,UFX+n) * rhoInv;
     }
 #endif
 

Exec/hydro_tests/rotating_torus/GNUmakefile

@@ -20,7 +20,7 @@ CASTRO_HOME  ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := polytrope
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/test_convect/GNUmakefile

@@ -18,7 +18,7 @@ USE_MODEL_PARSER = TRUE
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/toy_convect/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = hotcno.net
 

Exec/reacting_tests/bubble_convergence/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := triple_alpha_plus_cago
 
 PROBLEM_DIR ?= ./

Exec/reacting_tests/nse_test/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := aprox19
 USE_NSE_TABLE := TRUE
 

Exec/reacting_tests/reacting_bubble/GNUmakefile

@@ -17,7 +17,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := ignition_simple
 
 PROBLEM_DIR ?= ./

Exec/science/Detonation/GNUmakefile

@@ -16,7 +16,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 ifeq ($(USE_NSE_NET), TRUE)
 	NETWORK_DIR := subch_base
 	SCREEN_METHOD := chabrier1998

Exec/science/Detonation/GNUmakefile.nse

@@ -17,7 +17,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := aprox19
 USE_NSE_TABLE = TRUE
 

Exec/science/nova/GNUmakefile

@@ -21,7 +21,7 @@ MAX_NPTS_MODEL = 20000
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := nova2
 
 # Thi sets the conductivity EOS directory in $(MICROPHYSICS_HOME)/conductivity

Exec/science/subch_planar/GNUmakefile

@@ -0,0 +1,35 @@
+PRECISION        = DOUBLE
+PROFILE          = FALSE
+DEBUG            = FALSE
+DIM              = 2
+
+COMP	         = gnu
+
+USE_MPI          = TRUE
+USE_OMP          = FALSE
+
+USE_GRAV         = TRUE
+USE_REACT        = TRUE
+
+USE_SHOCK_VAR 	 = TRUE
+
+MAX_NPTS_MODEL 	 = 32678
+
+USE_MODEL_PARSER = TRUE
+
+USE_SIMPLIFIED_SDC = TRUE
+
+CASTRO_HOME ?= ../../..
+
+# This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
+EOS_DIR     := helmholtz
+
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
+NETWORK_DIR := subch_simple
+
+PROBLEM_DIR ?= ./
+
+Bpack   := $(PROBLEM_DIR)/Make.package
+Blocs   := $(PROBLEM_DIR)
+
+include $(CASTRO_HOME)/Exec/Make.Castro

Exec/science/subch_planar/Make.package

@@ -0,0 +1 @@
+

Exec/science/subch_planar/Problem_Derive.H

@@ -0,0 +1,10 @@
+
+void ca_dergradpoverp
+  (const amrex::Box& bx, amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/,
+   const amrex::FArrayBox& datfab, const amrex::Geometry& geomdata,
+   amrex::Real /*time*/, const int* /*bcrec*/, int /*level*/);
+
+void ca_dergradpoverp1
+  (const amrex::Box& bx, amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/,
+   const amrex::FArrayBox& datfab, const amrex::Geometry& geomdata,
+   amrex::Real /*time*/, const int* /*bcrec*/, int /*level*/);