Merge branch 'whannah/mmf/pam-impl' (#5906)

ntegration of new CRM for MMF, referred to as the "portable atmosphere model" or PAM, that uses the P3 and SHOC physics implementation from SCREAM/EAMxx. The new CRM is used by the compsets with "MMF2" in their name. While the PAM code is isolated to a submodule, several large changes outside were required to facilitate this integration. The PAM interface code is a substantial part of this, but there was also new conflicts created by EAMv3 integration efforts which led to the creation of various "dummy" modules to avoid building the conflicting modules, which are unnecessary for MMF. A new integration is added to test the new CRM => SMS_Ln3.ne4pg2_ne4pg2.F2010-MMF2 A few things to note about the current MMF2 implementation - When running with PAM on the GPU the use of openMP threads is problematic for reasons that are unclear. This seems fixable, but more work is needed to look into this. For now, running on GPUs without OMP threads is sufficient, and CPU cases are not affected by this problem - Although some offline PAM tests have used longer time steps, when running within the MMF configuration at ne30 the PAM time step needs to be 4 seconds for stability. A lot of performance can be gained by increasing this time step, but more work is needed to understand the source of the instability. - PAM performance also seems highly machine dependent. For example, using 22 nodes on Perlmutter runs 6x faster than 64 nodes on Summit for the same case. [BFB]
E3SM-Project · Oct 10, 2023 · 732e699 · 732e699
2 parents e195c80 + ddf0c37
commit 732e699
Show file tree

Hide file tree

Showing 44 changed files with 3,653 additions and 203 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -70,6 +70,9 @@
 	path = components/ww3/src/WW3
 	url = [email protected]:E3SM-Project/WW3.git
 	branch = e3sm
+[submodule "components/eam/src/physics/crm/pam/external"]
+	path = components/eam/src/physics/crm/pam/external
+	url = [email protected]:E3SM-Project/PAM.git
 [submodule "externals/haero"]
 	path = externals/haero
 	url = [email protected]:eagles-project/haero.git

diff --git a/cime_config/machines/cmake_macros/gnu.cmake b/cime_config/machines/cmake_macros/gnu.cmake
@@ -1,6 +1,9 @@
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64")
   string(APPEND CFLAGS "-mcmodel=small")
   string(APPEND FFLAGS "-mcmodel=small")
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
+  string(APPEND CFLAGS " -mcmodel=large")
+  string(APPEND FFLAGS " -mcmodel=large")
 else()
   string(APPEND CFLAGS "-mcmodel=medium")
   string(APPEND FFLAGS "-mcmodel=medium")

diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake
@@ -21,7 +21,7 @@ string(APPEND CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
 string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
 string(APPEND SLIBS    " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ")
 
-string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off")
 
 set(USE_HIP "TRUE")
 string(APPEND HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip")
diff --git a/cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake b/cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake
@@ -7,6 +7,7 @@ endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND SLIBS " -lblas -llapack")
 string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off")
 if (NOT DEBUG)
   string(APPEND CFLAGS " -O2")
   string(APPEND FFLAGS " -O2")

diff --git a/cime_config/machines/cmake_macros/gnugpu_summit.cmake b/cime_config/machines/cmake_macros/gnugpu_summit.cmake
@@ -14,3 +14,4 @@ string(APPEND SLIBS " -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_R
 set(MPICXX "mpiCC")
 set(PIO_FILESYSTEM_HINTS "gpfs")
 set(USE_CUDA "TRUE")
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_POWER9=On -DKokkos_ARCH_VOLTA70=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_OPENMP=Off")
diff --git a/cime_config/tests.py b/cime_config/tests.py
@@ -339,6 +339,7 @@
             "ERP_Ln9.ne4pg2_oQU480.WCYCL20TRNS-MMF1.allactive-mmf_fixed_subcycle",
             "ERS_Ln9.ne4pg2_ne4pg2.FRCE-MMF1.eam-cosp_nhtfrq9",
             "SMS_Ln5.ne4_ne4.FSCM-ARM97-MMF1",
+            "SMS_Ln3.ne4pg2_ne4pg2.F2010-MMF2",
             )
         },
 

diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake
@@ -116,6 +116,22 @@ function(build_model COMP_CLASS COMP_NAME)
                              cmake/atm/../../eam/src/physics/crm/crm_ecpp_output_module.F90 )
     endif()
 
+    if (USE_PAM)
+      message(STATUS "Building PAM")
+      # PAM_HOME is where the samxx source code lives
+      set(PAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/crm/pam)
+      # PAM_BIN is where the samxx library will live
+      set(PAM_BIN  ${CMAKE_CURRENT_BINARY_DIR}/pam)
+      # Build the static samxx library
+      add_subdirectory(${PAM_HOME} ${PAM_BIN})
+      # Add samxx F90 files to the main E3SM build
+      set(SOURCES ${SOURCES} cmake/atm/../../eam/src/physics/crm/pam/params.F90
+                             cmake/atm/../../eam/src/physics/crm/crm_ecpp_output_module.F90
+                             cmake/atm/../../eam/src/physics/crm/pam/pam_driver.F90) 
+      # Pam interface need to include modules from pam
+      include_directories(${PAM_BIN}/external/pam_core)
+    endif()
+
     # Add rrtmgp++ source code if asked for
     if (USE_RRTMGPXX)
       message(STATUS "Building RRTMGPXX")
@@ -245,6 +261,9 @@ function(build_model COMP_CLASS COMP_NAME)
       if (USE_SAMXX)
         target_link_libraries(${TARGET_NAME} PRIVATE samxx)
       endif()
+      if (USE_PAM)
+        target_link_libraries(${TARGET_NAME} PRIVATE pam_driver)
+      endif()
       if (USE_RRTMGPXX)
         target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface)
       endif()

diff --git a/components/cmake/common_setup.cmake b/components/cmake/common_setup.cmake
@@ -35,14 +35,22 @@ if (NOT HAS_SAMXX EQUAL -1)
   set(USE_SAMXX TRUE)
 endif()
 
+# Look for -crm pam in the CAM_CONFIG_OPTS CIME variable
+# If it's found, then enable USE_PAM
+string(FIND "${CAM_CONFIG_OPTS}" "-crm pam" HAS_PAM)
+if (NOT HAS_PAM EQUAL -1)
+  # The following is for the PAM code:
+  set(USE_PAM TRUE)
+endif()
+
 string(FIND "${CAM_CONFIG_OPTS}" "-rrtmgpxx" HAS_RRTMGPXX)
 if (NOT HAS_RRTMGPXX EQUAL -1)
   # The following is for the RRTMGPXX code:
   set(USE_RRTMGPXX TRUE)
 endif()
 
 # If samxx or rrtmgpxx is being used, then YAKL must be used as well
-if (USE_SAMXX OR USE_RRTMGPXX)
+if (USE_SAMXX OR USE_RRTMGPXX OR USE_PAM)
     set(USE_YAKL TRUE)
 else()
     set(USE_YAKL FALSE)

diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist
@@ -1705,13 +1705,9 @@ if ( !($phys eq 'ideal' or $phys eq 'adiabatic') ) {
 # Cloud optics; for MMF be specific about which optics package to use
 if ($rad_pkg eq 'rrtmg' or $rad_pkg eq 'rrtmgp') {
     if ($cfg->get('use_MMF')) {
-        if ($cfg->get('MMF_microphysics_scheme') eq 'sam1mom') {
-            add_default($nl, 'liqcldoptics', 'val' => 'slingo');
-            add_default($nl, 'icecldoptics', 'val' => 'ebertcurry');
-        } else {
-            add_default($nl, 'liqcldoptics', 'val' => 'gammadist');
-            add_default($nl, 'icecldoptics', 'val' => 'mitchell');
-        }
+        # NOTE: eventually we should switch PAM optics to gammadist+mitchell
+        add_default($nl, 'liqcldoptics', 'val' => 'slingo');
+        add_default($nl, 'icecldoptics', 'val' => 'ebertcurry');
     } else {
         add_default($nl, 'liqcldoptics');
         add_default($nl, 'icecldoptics');

diff --git a/components/eam/bld/config_files/definition.xml b/components/eam/bld/config_files/definition.xml
@@ -345,15 +345,15 @@ Switch to enable explicit scalar momentum transport for 2D CRM in E3SM-MMF: 0=of
 <entry id="use_ECPP" valid_values="0,1" value="0">
 Switch to enable or disable ECPP in CAM: 0=off, 1=on.
 </entry>
-<entry id="use_crm_cldfrac" valid_values="0,1" value="0">
-Switch to enable or disable fractional cloudiness in m2005 microphysics in CRM: 0=off, 1=on.
-</entry>
-<entry id="MMF_microphysics_scheme" valid_values="sam1mom,m2005" value="sam1mom">
+<entry id="MMF_microphysics_scheme" valid_values="sam1mom,p3" value="sam1mom">
 MMF CRM cloud microphysics scheme
 </entry>
-<entry id="crm" valid_values="sam,samomp,samxx" value="sam">
+<entry id="crm" valid_values="sam,samomp,samxx,pam" value="sam">
 MMF CRM model
 </entry>
+<entry id="pam_dycor" valid_values="awfl,spam" value="spam">
+MMF PAM dycor option
+</entry>
 <entry id="crm_adv" valid_values="MPDATA,UM5" value="MPDATA">
 MMF CRM advection scheme
 </entry>

diff --git a/components/eam/bld/configure b/components/eam/bld/configure
@@ -272,20 +272,24 @@ OPTIONS
 
   Options for MMF:
 
-     -use_MMF           Build the MMF configuration (super-parameterized EAM)
-     -use_MMF_VT        enable CRM variance transport (VT)
-     -use_MMF_ESMT      enable CRM explicit scalar momentum transport (ESMT)
-     -crm_nx <n>        CRM's x-grid.
-     -crm_ny <n>        CRM's y-grid.
-     -crm_nz <n>        CRM's z-grid.
-     -crm_dx <n>        CRM's horizontal grid spacing.
-     -crm_dt <n>        CRM's timestep.
-     -crm_nx_rad <n>    number of averaged columns for CRM radiation calculation in x direction
-     -crm_ny_rad <n>    number of averaged columns for CRM radiation calculation in y direction
-     -MMF_microphysics_scheme <string>   CRM microphysics package name [sam1mom | m2005 ].
-     -use_ECPP          use CRM clouds for vertical transport, aqueous chemistry and wet removable of aerosols
-     -crm_adv           CRM advection scheme [MPDATA | UM5]
-     -crm <model>       CRM model [sam | samomp | samxx]
+     -use_MMF                  Build the MMF configuration (super-parameterized EAM)
+     -use_MMF_VT               enable CRM variance transport (VT)
+     -use_MMF_ESMT             enable CRM explicit scalar momentum transport (ESMT)
+     -use_ECPP                 enable explicit-clouds-parameterized-pollutants scheme
+     -crm_nx <n>               CRM's x-grid.
+     -crm_ny <n>               CRM's y-grid.
+     -crm_nz <n>               CRM's z-grid.
+     -crm_dx <n>               CRM's horizontal grid spacing.
+     -crm_dt <n>               CRM's timestep.
+     -crm_nx_rad <n>           # of columns for CRM radiation calculation in x direction
+     -crm_ny_rad <n>           # of columns for CRM radiation calculation in y direction
+     -MMF_microphysics_scheme  CRM micro scheme - depends on CRM model [ sam1mom | p3 ].
+     -crm                      MMF CRM model            [ sam | samxx | pam]
+     -crm_adv                  SAM CRM advection scheme [ MPDATA | UM5]
+     -pam_dycor                PAM CRM dycor option     [ awfl | spam ]
+
+   Options for radiation:
+
      -rrtmgpxx          Use RRTMGP++ code
 EOF
 }
@@ -358,9 +362,10 @@ GetOptions(
     "crm_dt=s"                  => \$opts{'crm_dt'},
     "crm_nx_rad=s"              => \$opts{'crm_nx_rad'},
     "crm_ny_rad=s"              => \$opts{'crm_ny_rad'},
-    "MMF_microphysics_scheme=s"     => \$opts{'MMF_microphysics_scheme'},
+    "MMF_microphysics_scheme=s" => \$opts{'MMF_microphysics_scheme'},
     "crm_adv=s"                 => \$opts{'crm_adv'},
     "crm=s"                     => \$opts{'crm'},
+    "pam_dycor=s"               => \$opts{'pam_dycor'},
     "rrtmgpxx"                  => \$opts{'rrtmgpxx'},
     "debug"                     => \$opts{'debug'},
     "rain_evap_to_coarse_aero"  => \$opts{'rain_evap_to_coarse_aero'},
@@ -1293,6 +1298,7 @@ if (defined $opts{'use_MMF'}) {
     if (defined $opts{'crm_nx_rad'}) { $cfg_ref->set('crm_nx_rad', $opts{'crm_nx_rad'}); }
     if (defined $opts{'crm_ny_rad'}) { $cfg_ref->set('crm_ny_rad', $opts{'crm_ny_rad'}); }
     if (defined $opts{'crm_adv'}) { $cfg_ref->set('crm_adv', $opts{'crm_adv'}); }
+    if (defined $opts{'pam_dycor'})    { $cfg_ref->set('pam_dycor', $opts{'pam_dycor'}); }
     $cfg_ref->set('MMF_microphysics_scheme', $opts{'MMF_microphysics_scheme'});
     $cfg_ref->set('crm', $opts{'crm'});
     if (defined $opts{'use_MMF_VT'})   { $cfg_ref->set('use_MMF_VT',   1); }
@@ -1812,12 +1818,16 @@ if (defined $opts{'use_MMF'}) {
     my $crm_nx_rad = $cfg_ref->get('crm_nx_rad');
     my $crm_ny_rad = $cfg_ref->get('crm_ny_rad');
     my $crm = $cfg_ref->get('crm');
+    my $pam_dycor = $cfg_ref->get('pam_dycor');
     my $MMF_microphysics_scheme = $cfg_ref->get('MMF_microphysics_scheme');
     my $crm_adv = $cfg_ref->get('crm_adv');
     my $yes3Dval = 1;
     if ($crm_ny eq 1) {$yes3Dval = 0;}
     if (defined $opts{'crm_adv'}) { $cfg_cppdefs .= " -D_$crm_adv "; }
-    $cfg_cppdefs .= " -D$MMF_microphysics_scheme -DYES3DVAL=$yes3Dval";
+    $cfg_cppdefs .= " -DYES3DVAL=$yes3Dval";
+    if ($MMF_microphysics_scheme eq 'sam1mom') {
+        $cfg_cppdefs .= " -D$MMF_microphysics_scheme ";
+    }
     $cfg_cppdefs .= " -DCRM_NX=$crm_nx -DCRM_NY=$crm_ny -DCRM_NZ=$crm_nz ";
     $cfg_cppdefs .= " -DCRM_DX=$crm_dx -DCRM_DT=$crm_dt ";
     if (defined $opts{'use_ECPP'}) { $cfg_cppdefs .= " -DECPP " }
@@ -1832,13 +1842,14 @@ if (defined $opts{'use_MMF'}) {
         $cfg_cppdefs .= " -DCRM_NY_RAD=$crm_ny "
     }
     if ($crm_ny eq 1) {$yes3Dval = 0;}
-    my $crm = $cfg_ref->get('crm');
     if    ($crm eq 'sam') {
         $cfg_cppdefs .= " -DMMF_SAM "
-    } elsif ($crm eq 'samomp') {
-        $cfg_cppdefs .= " -DMMF_SAMOMP"
     } elsif ($crm eq 'samxx') {
         $cfg_cppdefs .= " -DMMF_SAMXX "
+    } elsif ($crm eq 'pam') {
+        $cfg_cppdefs .= " -DMMF_PAM ";
+        if ($pam_dycor eq 'awfl') { $cfg_cppdefs .= " -DMMF_PAM_DYCOR_AWFL " }
+        if ($pam_dycor eq 'spam') { $cfg_cppdefs .= " -DMMF_PAM_DYCOR_SPAM " }
     }
 
 }
@@ -2555,6 +2566,7 @@ sub write_filepath
     my $ice           = $cfg_ref->get('ice');
     my $rof           = $cfg_ref->get('rof');
     my $caseroot      = $cfg_ref->get('caseroot');
+    my $crm           = $cfg_ref->get('crm');
 
     # Root directory
     my $camsrcdir = "$cam_root/components";
@@ -2589,6 +2601,7 @@ sub write_filepath
 
         # interface and base model code
         print $fh "$camsrcdir/eam/src/physics/crm\n";
+        print $fh "$camsrcdir/eam/src/physics/crm/dummy_modules\n";
 
         # MMF-specific radiation drivers (overrides default drivers that exist
         # in each individual radiation package; done this way to avoid
@@ -2603,7 +2616,6 @@ sub write_filepath
             print $fh "$camsrcdir/eam/src/physics/crm/ecpp\n";
         }
 
-        my $crm = $cfg_ref->get('crm');
         if ($crm eq 'sam') {
           #####################################################
           ## -crm sam will GLOB directories for CMake
@@ -2614,41 +2626,9 @@ sub write_filepath
           my $MMF_microphysics_scheme = $cfg_ref->get('MMF_microphysics_scheme');
           if ($MMF_microphysics_scheme eq 'sam1mom') {
               print $fh "$camsrcdir/eam/src/physics/crm/sam/MICRO_SAM1MOM\n";
-          } elsif ($MMF_microphysics_scheme eq 'm2005') {
-              print $fh "$camsrcdir/eam/src/physics/crm/sam/MICRO_M2005\n";
           }
-
           # turbulence closure
           print $fh "$camsrcdir/eam/src/physics/crm/sam/SGS_TKE\n";
-
-          # advection scheme
-          my $crm_adv = 'MPDATA';
-          if (defined $opts{'crm_adv'}) { $crm_adv = $cfg_ref->get('crm_adv'); }
-          if ($crm_adv eq 'MPDATA') {
-            print $fh "$camsrcdir/eam/src/physics/crm/sam/ADV_MPDATA\n";
-          } elsif ($crm_adv eq 'UM5') {
-            print $fh "$camsrcdir/eam/src/physics/crm/sam/ADV_UM5\n";
-          } else {
-            die "configure ERROR: invalid option - crm_adv: $crm_adv.\n";
-          }
-
-        } elsif ($crm eq 'samomp') {
-          #####################################################
-          ## -crm samomp will GLOB directories for CMake
-          ######################################################
-          print $fh "$camsrcdir/eam/src/physics/crm/samomp\n";
-          #
-          # microphysics
-          my $MMF_microphysics_scheme = $cfg_ref->get('MMF_microphysics_scheme');
-          if ($MMF_microphysics_scheme eq 'sam1mom') {
-              print $fh "$camsrcdir/eam/src/physics/crm/samomp/MICRO_SAM1MOM\n";
-          } elsif ($MMF_microphysics_scheme eq 'm2005') {
-              print $fh "$camsrcdir/eam/src/physics/crm/samomp/MICRO_M2005\n";
-          }
-
-          # turbulence closure
-          print $fh "$camsrcdir/eam/src/physics/crm/samomp/SGS_TKE\n";
-
           # advection scheme
           my $crm_adv = 'MPDATA';
           if (defined $opts{'crm_adv'}) { $crm_adv = $cfg_ref->get('crm_adv'); }
@@ -2659,12 +2639,6 @@ sub write_filepath
           } else {
             die "configure ERROR: invalid option - crm_adv: $crm_adv.\n";
           }
-
-        } elsif ($crm eq 'samxx') {
-          #####################################################
-          ## -crm samxx will add_subdirectory in CMake rather
-          ##            than GLOBing.
-          #####################################################
         }
 
     }
@@ -2725,11 +2699,12 @@ sub write_filepath
        print $fh "$camsrcdir/eam/src/physics/silhs\n";
     }
 
-    # set Fortran P3 path, needed even if microphys is not P3
-    if ($is_scream_config eq '1') {
-       print $fh "$camsrcdir/eam/src/physics/p3/scream\n";
-    } else {
-       print $fh "$camsrcdir/eam/src/physics/p3/eam\n";
+    if (not(defined $opts{'use_MMF'})) {
+       if ($is_scream_config eq '1') {
+          print $fh "$camsrcdir/eam/src/physics/p3/scream\n";
+       } else {
+          print $fh "$camsrcdir/eam/src/physics/p3/eam\n";
+       }
     }
 
     print $fh "$camsrcdir/eam/src/dynamics/$dyn\n";
@@ -2867,6 +2842,7 @@ sub write_cppdefs
     my  $fh = new IO::File;
 
     my $dyn           = $cfg_ref->get('dyn');
+    my $crm           = $cfg_ref->get('crm');
 
     print "Writing CPPDEFS for EAM in $file, with $dyn and $opts{'dyn_target'}\n";