add configure option rrtmgp_smp; misc cleanup

brian-eaton · Jul 19, 2023 · fd417ba · fd417ba
1 parent 3022b4c
commit fd417ba
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 37 deletions.
diff --git a/bld/configure b/bld/configure
@@ -103,7 +103,7 @@ OPTIONS
      -prog_species <list>Comma-separate list of prognostic mozart species packages.
                         Currently available: DST,SSLT,SO4,GHG,OC,BC,CARBON16
      -psubcols <n>      Maximum number of sub-columns in a run - set to 1 if not using sub-columns (default)
-     -rad <name>        Specify the radiation package [rrtmg | rrtmgp | camrt]
+     -rad <name>        Specify the radiation package [rrtmg | rrtmgp | rrtmgp_smp | camrt]
      -silhs             Switch on SILHS.
      -spcam_clubb_sgs   Turn on the SPCAM version of CLUBB
      -spcam_nx <n>      SPCAM x-grid. - defaults to 4 (note the CRM requires spcam_nx to be greater than or equal to 4)
@@ -1066,8 +1066,16 @@ elsif ($phys_pkg =~ m/^cam[56]$|^cam_dev$|^spcam_m2005$/) {
     $rad_pkg = 'rrtmg';
 }
 # Allow the user to override the default via the commandline.
+my $use_rrtmgp_smp = 0;
 if (defined $opts{'rad'}) {
     $rad_pkg = lc($opts{'rad'});
+    # If the radiation package is set to rrtmgp_smp then will add the smp code version
+    # (openmp and openacc) to the Filepath file, but strip off the "_smp" when setting
+    # the radiation package name in the config_cache file.
+    if ($rad_pkg eq 'rrtmgp_smp') {
+        $use_rrtmgp_smp = 1;
+        $rad_pkg =~ s!_smp!!
+    }
 }
 
 # consistency checks...
@@ -2180,11 +2188,14 @@ sub write_filepath
     }
     elsif ($rad eq 'rrtmgp') {
         print $fh "$camsrcdir/src/physics/rrtmgp\n";
-        print $fh "$camsrcdir/src/physics/rrtmgp/ext\n";
         print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp\n";
         print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte\n";
-        print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte/kernels\n";
+        if ($use_rrtmgp_smp) {
+            print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp/kernels-openacc\n";
+            print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte/kernels-openacc\n";
+        }
         print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp/kernels\n";
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte/kernels\n";
         print $fh "$camsrcdir/src/physics/rrtmgp/ext/extensions\n";
         print $fh "$camsrcdir/src/physics/rrtmgp/ext/extensions/cloud_optics\n";
     }

diff --git a/bld/namelist_files/use_cases/1850_cam5.xml b/bld/namelist_files/use_cases/1850_cam5.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+
+<namelist_defaults>
+
+<!-- Solar constant from Lean (via Caspar Ammann) -->
+<solar_data_file>atm/cam/solar/SOLAR_SPECTRAL_Lean_1610-2008_annual_c090324.nc</solar_data_file>
+<solar_data_ymd>18500101</solar_data_ymd>
+<solar_data_type>FIXED</solar_data_type>
+
+<!-- 1850 GHG values from ipcc-ar4-cfcs-1850-2000.xls (Doug Kinnison) -->
+<co2vmr>284.7e-6</co2vmr>
+<ch4vmr>791.6e-9</ch4vmr>
+<n2ovmr>275.68e-9</n2ovmr>
+<f11vmr>12.48e-12</f11vmr>
+<f12vmr>0.0</f12vmr>
+
+<!-- 1850 ozone data is from Jean-Francois Lamarque -->
+<prescribed_ozone_datapath>atm/cam/ozone</prescribed_ozone_datapath>
+<prescribed_ozone_file>ozone_1.9x2.5_L26_1850clim_c090420.nc</prescribed_ozone_file>
+<prescribed_ozone_name>O3</prescribed_ozone_name>
+<prescribed_ozone_type>CYCLICAL</prescribed_ozone_type>
+<prescribed_ozone_cycle_yr>1850</prescribed_ozone_cycle_yr>
+
+<!-- Surface emissions for MAM -->
+<srf_emis_type>CYCLICAL</srf_emis_type>
+<dms_emis_file	   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/aerocom_mam3_dms_surf_2000_c090129.nc</dms_emis_file>
+<so2_emis_file	   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_so2_surf_1850_c090726.nc</so2_emis_file>
+<soag_emis_file	   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_soag_1.5_surf_1850_c100217.nc</soag_emis_file>
+<bc_a1_emis_file   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_bc_surf_1850_c090726.nc</bc_a1_emis_file>
+<pom_a1_emis_file  chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_oc_surf_1850_c090726.nc</pom_a1_emis_file>
+<so4_a1_emis_file  chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_so4_a1_surf_1850_c090726.nc</so4_a1_emis_file>
+<so4_a2_emis_file  chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_so4_a2_surf_1850_c090726.nc</so4_a2_emis_file>
+<num_a1_emis_file  chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_num_a1_surf_1850_c090726.nc</num_a1_emis_file>
+<num_a2_emis_file  chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_num_a2_surf_1850_c090726.nc</num_a2_emis_file>
+
+<!-- External forcing for BAM or MAM -->
+<ext_frc_type>CYCLICAL</ext_frc_type>
+<so2_ext_file      chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_so2_elev_1850_c090726.nc </so2_ext_file>
+<so4_a1_ext_file   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_so4_a1_elev_1850_c090726.nc</so4_a1_ext_file>
+<so4_a2_ext_file   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_so4_a2_elev_1850_c090726.nc</so4_a2_ext_file>
+<num_a1_ext_file   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_num_a1_elev_1850_c090726.nc</num_a1_ext_file>
+<num_a2_ext_file   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_num_a2_elev_1850_c090726.nc</num_a2_ext_file>
+<pom_a1_ext_file   chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_oc_elev_1850_c090726.nc </pom_a1_ext_file>
+<bc_a1_ext_file    chem="trop_mam3">atm/cam/chem/trop_mozart_aero/emis/ar5_mam3_bc_elev_1850_c090726.nc </bc_a1_ext_file>
+
+<!-- Prescribed oxidants for aerosol chemistry -->
+<tracer_cnst_type>CYCLICAL</tracer_cnst_type>
+<tracer_cnst_cycle_yr>1850</tracer_cnst_cycle_yr>
+<tracer_cnst_file>oxid_1.9x2.5_L26_1850clim_c091123.nc</tracer_cnst_file>
+
+<!-- sim_year used for CLM datasets and SSTs forcings -->
+<sim_year>1850</sim_year>
+
+</namelist_defaults>
diff --git a/src/chemistry/utils/solar_data.F90 b/src/chemistry/utils/solar_data.F90
@@ -91,6 +91,7 @@ subroutine solar_data_readnl( nlfile )
        write(iulog,*) 'solar_data_readnl: solar_data_type = ',trim(solar_data_type)
        write(iulog,*) 'solar_data_readnl: solar_data_ymd  = ',solar_data_ymd
        write(iulog,*) 'solar_data_readnl: solar_data_tod  = ',solar_data_tod
+       write(iulog,*) 'solar_data_readnl: solar_htng_spctrl_scl  = ',solar_htng_spctrl_scl
     endif
 
     solar_parms_on = solar_parms_data_file.ne.'NONE'

diff --git a/src/physics/rrtmg/radiation.F90 b/src/physics/rrtmg/radiation.F90
@@ -1215,12 +1215,6 @@ subroutine radiation_tend( &
                rd%aer_tau400(:ncol,:)       = aer_tau(:ncol,:,idx_sw_diag+1)
                rd%aer_tau700(:ncol,:)       = aer_tau(:ncol,:,idx_sw_diag-1)
 
-               print*,'--- Right before rad_rrtmg_sw ---'
-               do k=1,pver
-                  print '("LEVEL",i2,3x,"TAU (max) = ",f7.4,3x)', k,MAXVAL(c_cld_tau(:,1,k))
-               end do
-
-
                call rad_rrtmg_sw( &
                   lchnk, ncol, num_rrtmg_levs, r_state, state%pmid,          &
                   cldfprime, aer_tau, aer_tau_w, aer_tau_w_g,  aer_tau_w_f,  &

diff --git a/src/physics/rrtmgp/radiation.F90 b/src/physics/rrtmgp/radiation.F90
@@ -196,14 +196,18 @@ module radiation
 ! Number of layers in radiation calculations.
 integer :: nlay
 
-! Indices for copying data between cam and rrtmgp arrays
-! The code currently assumes the rrtmgp vertical index goes bottom to top,
-! while CAM goes top-to-bottom ... 
-! Newer RRTMGP checks for host model order and adjusts, so a lot of the assumptions are unncessary.
-integer :: ktopcamm ! cam index of top layer
-integer :: ktopradm ! rrtmgp index of layer corresponding to ktopcamm
-integer :: ktopcami ! cam index of top interface
-integer :: ktopradi ! rrtmgp index of interface corresponding to ktopcami
+! Indices for copying data between CAM/WACCM and RRTMGP arrays.  Since RRTMGP is
+! vertical order agnostic we can send data using the top to bottom order used
+! in CAM/WACCM.  But the number of layers that RRTMGP does computations for
+! may not match the number of layers in CAM/WACCM for two reasons:
+! 1. If the CAM model top is below 1 Pa, then RRTMGP does calculations for an
+!    extra layer that is added between 1 Pa and the model top.
+! 2. If the WACCM model top is above 1 Pa, then RRMTGP only does calculations
+!    for those model layers that are below 1 Pa.
+integer :: ktopcamm ! index in CAM arrays of top layer at which RRTMGP is active
+integer :: ktopcami ! index in CAM arrays of top interface at which RRTMGP is active
+integer :: ktopradm ! index in RRTMGP arrays of layer corresponding to CAM top layer
+integer :: ktopradi ! index in RRTMGP arrays of interface corresponding to CAM top interface
 
 ! LW coefficients
 type(ty_gas_optics_rrtmgp) :: kdist_lw ! bpm changed here
@@ -287,7 +291,7 @@ subroutine radiation_readnl(nlfile)
    call mpi_bcast(rrtmgp_coefs_lw_file, cl, mpi_character, mstrid, mpicom, ierr)
    if (ierr /= 0) call endrun(subroutine_name//": FATAL: mpi_bcast: rrtmgp_coefs_lw_file")
    call mpi_bcast(rrtmgp_coefs_sw_file, cl, mpi_character, mstrid, mpicom, ierr)
-   if (ierr /= 0) call endrun(subroutine_name//": FATAL: mpi_bcast: coefs_sw_file")
+   if (ierr /= 0) call endrun(subroutine_name//": FATAL: mpi_bcast: rrtmgp_coefs_sw_file")
    call mpi_bcast(iradsw, 1, mpi_integer, mstrid, mpicom, ierr)
    if (ierr /= 0) call endrun(subroutine_name//": FATAL: mpi_bcast: iradsw")
    call mpi_bcast(iradlw, 1, mpi_integer, mstrid, mpicom, ierr)
@@ -493,36 +497,26 @@ subroutine radiation_init(pbuf2d)
    character(len=*), parameter :: sub = 'radiation_init'
    !-----------------------------------------------------------------------
 
-   !
-   ! replacement of RRTMG's rrtmg_state_init
-   !
-
    ! Number of layers in radiation calculation is capped by the number of
    ! pressure interfaces below 1 Pa.  When the entire model atmosphere is
    ! below 1 Pa then an extra layer is added to the top of the model for
    ! the purpose of the radiation calculation.
    nlay = count( pref_edge(:) > 1._r8 ) ! pascals (0.01 mbar)
 
-   ! Use k*rad* to access variables ON THE RADIATION GRID
-   ! Use k*cam* to access variables ON THE CAM GRID
    if (nlay == pverp) then
-         ktopcamm = 1               ! interpretation: highest CAM grid layer at which radiation is active
-         ktopcami = 1 
-         ktopradm = nlay + 1 - pver ! radiation grid layer the corresponds to CAM's highest layer (expected to be 2)
-         ktopradi = nlay + 1 - pver
-   else ! nlay < pverp
-      ! nlay layers are set by radiation
-      ! nlay+1 interfaces are set by radiation
+      ! Top model interface is below 1 Pa.  RRTMGP is active in all model layers plus
+      ! 1 extra layer between model top and 1 Pa.
+      ktopcamm = 1
+      ktopcami = 1 
+      ktopradm = 2
+      ktopradi = 2
+   else
+      ! nlay < pverp.  nlay layers are set by radiation
       ktopcamm = pverp - nlay + 1
       ktopcami = pverp - nlay + 1
-      ktopradm = 1  ! radiation grid index at top is just 1
+      ktopradm = 1
       ktopradi = 1
    end if
-   ! bottom indices are known, so we don't need to have extra variables.
-   ! kbotcamm = pver
-   ! kbotcami = pverp
-   ! kbotradm = nlay
-   ! kbotradi = nlay + 1
 
    call set_available_gases(active_gases, available_gases) ! gases needed to initialize spectral info