Finalize 2.7.0 try2 (#606)

Co-authored-by: tomvothecoder <[email protected]>
E3SM-Project · Jul 11, 2022 · aedc079 · aedc079
1 parent 1fbb46b
commit aedc079
Show file tree

Hide file tree

Showing 12 changed files with 1,337 additions and 172 deletions.
diff --git a/README.md b/README.md
@@ -41,11 +41,11 @@ This diagnostics package is constructed for supporting the diagnostics task of D
 - fully integrates the functionality of NCAR's AMWG diagnostics package.
 - utilizes most updated observational datasets, including remote sensing, reanalysis and in-situ datasets.
 - interfaces with diagnostics developed from different E3SM science groups
-- is flexible for user specified diagnostics and being configured for use by other climate models.
+- is flexible for user specified diagnostics and being configured for use by other climate models (i.e. in CMIP format).
 
 ## Current State <a name="current-state"></a>
 
-Algorithm and visualization codes for the AMWG Set 5, 7, 4, 3, 13, 1, 14 diagnostics, namely lat-lon contour plots (Figure 1), polar contour plots (Figure 2), zonal mean 2d plots (Figure 3), zonal mean line plots (Figure 4), 2d joint histogram for COSP cloud simulator output (Figure 5), tables (Figure 6) and Taylor Diagrams (Figure 7) summarizing metrics, for climatology seasonal means, are implemented as core sets of this diagnostics package. More diagnostics harvested from E3SM v1 and v2 development are also supported.
+Algorithm and visualization codes for lat-lon contour plots (Figure 1), polar contour plots (Figure 2), zonal mean 2d plots (Figure 3), zonal mean line plots (Figure 4), 2d joint histogram for COSP cloud simulator output (Figure 5), tables (Figure 6) and Taylor Diagrams (Figure 7) summarizing metrics, for climatology seasonal means, are implemented as core sets of this diagnostics package. More diagnostics harvested from E3SM v1 and v2 development are also supported. Example of a complete set of diagnostics can be found [here](https://web.lcrc.anl.gov/public/e3sm/e3sm_diags_test_data/unit_test_complete_run/expected/all_sets/viewer/)
 
 The package features built-in user diagnostics, by specifying user desired diagnostics regions and pressure levels for variables with the vertical dimension.
 
@@ -72,6 +72,15 @@ In addition to default model versus observation comparison, the package also pro
 <img src="misc/example_fig7.png" alt="Figure7" style="width: 280px;"/>
 <h5 align="center">Figure 7: An example of Taylor diagram summarizing metrics calculated based on lat-lon contour plots diagnostics of several key variables</h5>
 
+## Reference
+
+Zhang, Chengzhu, Jean-Christophe Golaz, Ryan Forsyth, Tom Vo, Shaocheng Xie, Zeshawn Shaheen, Gerald L. Potter et al. "The E3SM Diagnostics Package (E3SM Diags v2): A Python-based Diagnostics Package for Earth System Models Evaluation." Geoscientific Model Development Discussions (2022): 1-35.
+
+
+## Acknowledgement
+
+The work is performed for the [E3SM](https://e3sm.org/) project, which is sponsored by Earth System Model Development ([ESMD](https://climatemodeling.science.energy.gov/program/earth-system-model-development)) program.  ESMD is a program area for the Earth and Environmental Systems Sciences Division ([EESSD](https://science.osti.gov/ber/Research/eessd)) in the Office of Biological and Environmental Research ([BER](https://science.osti.gov/ber)) within the [Department of Energy](https://www.energy.gov/)'s [Office of Science](https://science.osti.gov).
+
 ## License
 
 Copyright (c) 2018-2021, Energy Exascale Earth System Model Project

diff --git a/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py b/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py
@@ -0,0 +1,49 @@
+import cdms2
+from pathlib import Path
+import MV2
+
+datapath='/p/user_pub/e3sm/e3sm_diags_data/obs_for_e3sm_diags/climatology/TRMM-3B43v-7_3hr/'
+for path in Path(datapath).rglob('*.nc'):
+    print(path.name)
+    print(path)
+    filename = path.name.split("/")[-1]
+    print(filename)
+
+    #filename ='TRMM-3B43v-7_3hr_ANN_199801_201312_climo.nc'
+    f_in = cdms2.open(datapath+filename)
+    var0 = f_in('pr')
+    var = f_in('pr')
+    lat = var0.getLatitude()
+    lon = var0.getLongitude()
+    nlat = len(lat)
+    var.id = 'pr'
+    # Fix units problem for obs4mip
+    #var = var / 3600.0 * 1000 / 1000.0
+
+    # Manually shift time by 1.5 hour to work around an ncclimo problem:
+    # The shift may be due to the assumption that ncclimo makes that times are encoded using the CESM/E3SM convention where times refer to the end of the timestep
+    print('var.time',var.getTime()[:])
+    for ilat in range(nlat):
+        print,'ilat= ',ilat
+        var[0,ilat,:] = var0[7,ilat,:]
+        var[1:8,ilat,:] = var0[0:7,ilat,:]
+    newtime=cdms2.createAxis(MV2.zeros(8))
+    newtime.id="time" #name of dimension
+    newtime.designateTime()  # tell cdms to add attributes that make it time
+    newtime.units = var0.getTime().units
+    newtime[:] = [x-0.0625 for x in var0.getTime()[:]]
+    var.setAxis(0,newtime)
+    print(newtime)
+    print(var.getTime())
+
+
+    f_out = cdms2.open(filename,'w')
+    f_out.write(var)
+    att_keys = list(f_in.attributes.keys())
+    att_dic = {}
+    for i in range(len(att_keys)):
+        att_dic[i]=att_keys[i],f_in.attributes[att_keys[i]]
+        to_out = att_dic[i]
+        setattr(f_out,to_out[0],to_out[1])
+    f_in.close()
+    f_out.close()
diff --git a/analysis_data_preprocess/fix_units_TRMM-3B43v-7_3hr.py b/analysis_data_preprocess/fix_units_TRMM-3B43v-7_3hr.py
diff --git a/e3sm_diags/derivations/acme.py b/e3sm_diags/derivations/acme.py
@@ -29,7 +29,7 @@ def aplusb(var1, var2, target_units=None):
     return var1 + var2
 
 
-def convert_units(var, target_units):
+def convert_units(var, target_units):  # noqa: C901
     """Converts units of var to target_units.
     var is a cdms.TransientVariable."""
 
@@ -40,6 +40,17 @@ def convert_units(var, target_units):
         var = 100.0 * var
     elif not hasattr(var, "units") and var.id == "AODVIS":
         var.units = target_units
+    elif not hasattr(var, "units") and var.id == "AODDUST":
+        var.units = target_units
+    elif var.id == "FAREA_BURNED":
+        var = var * 1e9
+        var.units = target_units
+    elif var.units == "gC/m^2":
+        var = var / 1000.0
+        var.units = target_units
+    elif var.id == "FLOODPLAIN_VOLUME" and target_units == "km3":
+        var = var / 1.0e9
+        var.units = target_units
     elif var.id == "AOD_550_ann":
         var.units = target_units
     elif var.id == "AOD_550":
@@ -67,9 +78,12 @@ def convert_units(var, target_units):
     elif var.id == "prw" and var.units == "cm":
         var = var * 10.0  # convert from 'cm' to 'kg/m2' or 'mm'
         var.units = target_units
-    elif var.units in ["gN/m^2/s", "gP/m^2/s", "gC/m^2/s"] and target_units == "*/day":
+    elif var.units in ["gC/m^2/s"] and target_units == "g*/m^2/day":
         var = var * 24 * 3600
         var.units = var.units[0:7] + "day"
+    elif var.units in ["gN/m^2/s", "gP/m^2/s"] and target_units == "mg*/m^2/day":
+        var = var * 24 * 3600 * 1000.0
+        var.units = "m" + var.units[0:7] + "day"
     elif var.units in ["gN/m^2/day", "gP/m^2/day", "gC/m^2/day"]:
         pass
     else:
@@ -180,6 +194,14 @@ def tauxy(taux, tauy):
     return var
 
 
+def fp_uptake(a, b):
+    """plant uptake of soil mineral N"""
+    var = a / b
+    var.units = "dimensionless"
+    var.long_name = "Plant uptake of soil mineral N"
+    return var
+
+
 def albedo(rsdt, rsut):
     """TOA (top-of-atmosphere) albedo, rsut / rsdt, unit is nondimension"""
     var = rsut / rsdt
@@ -1344,6 +1366,14 @@ def cosp_histogram_standardize(cld: "FileVariable"):
         ]
     ),
     "AODABS": OrderedDict([(("abs550aer",), rename)]),
+    "AODDUST": OrderedDict(
+        [
+            (
+                ("AODDUST",),
+                lambda aod: convert_units(rename(aod), target_units="dimensionless"),
+            )
+        ]
+    ),
     # Surface temperature: Degrees C
     # (Temperature of the surface (land/water) itself, not the air)
     "TS": OrderedDict([(("ts",), rename)]),
@@ -1405,18 +1435,17 @@ def cosp_histogram_standardize(cld: "FileVariable"):
     "SOILWATER_10CM": OrderedDict([(("mrsos",), rename)]),
     "SOILWATER_SUM": OrderedDict([(("mrso",), rename)]),
     "SOILICE_SUM": OrderedDict([(("mrfso",), rename)]),
-    "QOVER": OrderedDict([(("mrros",), rename)]),
     "QRUNOFF": OrderedDict(
         [
             (("QRUNOFF",), lambda qrunoff: qflxconvert_units(qrunoff)),
-            (("mrro",), rename),
+            (("mrro",), lambda qrunoff: qflxconvert_units(qrunoff)),
         ]
     ),
     "QINTR": OrderedDict([(("prveg",), rename)]),
     "QVEGE": OrderedDict(
         [
             (("QVEGE",), lambda qevge: qflxconvert_units(rename(qevge))),
-            (("evspsblveg",), rename),
+            (("evspsblveg",), lambda qevge: qflxconvert_units(rename(qevge))),
         ]
     ),
     "QVEGT": OrderedDict(
@@ -1427,33 +1456,167 @@ def cosp_histogram_standardize(cld: "FileVariable"):
     "QSOIL": OrderedDict(
         [
             (("QSOIL",), lambda qsoil: qflxconvert_units(rename(qsoil))),
-            (("evspsblsoi",), rename),
+            (("evspsblsoi",), lambda qsoil: qflxconvert_units(rename(qsoil))),
+        ]
+    ),
+    "QDRAI": OrderedDict(
+        [
+            (("QDRAI",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QINFL": OrderedDict(
+        [
+            (("QINFL",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QIRRIG_GRND": OrderedDict(
+        [
+            (("QIRRIG_GRND",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QIRRIG_ORIG": OrderedDict(
+        [
+            (("QIRRIG_ORIG",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QIRRIG_REAL": OrderedDict(
+        [
+            (("QIRRIG_REAL",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QIRRIG_SURF": OrderedDict(
+        [
+            (("QIRRIG_SURF",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QIRRIG_WM": OrderedDict(
+        [
+            (("QIRRIG_WM",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QOVER": OrderedDict(
+        [
+            (("QOVER",), lambda q: qflxconvert_units(rename(q))),
+            (("mrros",), lambda q: qflxconvert_units(rename(q))),
+        ]
+    ),
+    "QRGWL": OrderedDict(
+        [
+            (("QRGWL",), lambda q: qflxconvert_units(rename(q))),
         ]
     ),
     "RAIN": OrderedDict(
         [
             (("RAIN",), lambda rain: qflxconvert_units(rename(rain))),
         ]
     ),
+    "SNOW": OrderedDict(
+        [
+            (("SNOW",), lambda snow: qflxconvert_units(rename(snow))),
+        ]
+    ),
     "TRAN": OrderedDict([(("tran",), rename)]),
     "TSOI": OrderedDict([(("tsl",), rename)]),
     "LAI": OrderedDict([(("lai",), rename)]),
     # Additional land variables requested by BGC evaluation
+    "FAREA_BURNED": OrderedDict(
+        [
+            (
+                ("FAREA_BURNED",),
+                lambda v: convert_units(v, target_units="proportionx10^9"),
+            )
+        ]
+    ),
+    "FLOODPLAIN_VOLUME": OrderedDict(
+        [(("FLOODPLAIN_VOLUME",), lambda v: convert_units(v, target_units="km3"))]
+    ),
     "TLAI": OrderedDict([(("TLAI",), rename)]),
     "EFLX_LH_TOT": OrderedDict([(("EFLX_LH_TOT",), rename)]),
-    "GPP": OrderedDict([(("GPP",), lambda v: convert_units(v, target_units="*/day"))]),
-    "NBP": OrderedDict([(("NBP",), lambda v: convert_units(v, target_units="*/day"))]),
-    "NPP": OrderedDict([(("NPP",), lambda v: convert_units(v, target_units="*/day"))]),
-    "TOTVEGC": OrderedDict([(("TOTVEGC",), rename)]),
-    "TOTSOMC": OrderedDict([(("TOTSOMC",), rename)]),
+    "GPP": OrderedDict(
+        [(("GPP",), lambda v: convert_units(v, target_units="g*/m^2/day"))]
+    ),
+    "HR": OrderedDict(
+        [(("HR",), lambda v: convert_units(v, target_units="g*/m^2/day"))]
+    ),
+    "NBP": OrderedDict(
+        [(("NBP",), lambda v: convert_units(v, target_units="g*/m^2/day"))]
+    ),
+    "NPP": OrderedDict(
+        [(("NPP",), lambda v: convert_units(v, target_units="g*/m^2/day"))]
+    ),
+    "TOTVEGC": OrderedDict(
+        [(("TOTVEGC",), lambda v: convert_units(v, target_units="kgC/m^2"))]
+    ),
+    "TOTSOMC": OrderedDict(
+        [(("TOTSOMC",), lambda v: convert_units(v, target_units="kgC/m^2"))]
+    ),
     "TOTSOMN": OrderedDict([(("TOTSOMN",), rename)]),
     "TOTSOMP": OrderedDict([(("TOTSOMP",), rename)]),
     "FPG": OrderedDict([(("FPG",), rename)]),
     "FPG_P": OrderedDict([(("FPG_P",), rename)]),
     "TBOT": OrderedDict([(("TBOT",), rename)]),
-    "CPOOL": OrderedDict([(("CPOOL",), rename)]),
-    "SR": OrderedDict([(("SR",), rename)]),
+    "CPOOL": OrderedDict(
+        [(("CPOOL",), lambda v: convert_units(v, target_units="kgC/m^2"))]
+    ),
+    "LEAFC": OrderedDict(
+        [(("LEAFC",), lambda v: convert_units(v, target_units="kgC/m^2"))]
+    ),
+    "SR": OrderedDict([(("SR",), lambda v: convert_units(v, target_units="kgC/m^2"))]),
     "RH2M": OrderedDict([(("RH2M",), rename)]),
+    "DENIT": OrderedDict(
+        [(("DENIT",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "GROSS_NMIN": OrderedDict(
+        [(("GROSS_NMIN",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "GROSS_PMIN": OrderedDict(
+        [(("GROSS_PMIN",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "NDEP_TO_SMINN": OrderedDict(
+        [(("NDEP_TO_SMINN",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "NFIX_TO_SMINN": OrderedDict(
+        [(("NFIX_TO_SMINN",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "PLANT_NDEMAND_COL": OrderedDict(
+        [
+            (
+                ("PLANT_NDEMAND_COL",),
+                lambda v: convert_units(v, target_units="mg*/m^2/day"),
+            )
+        ]
+    ),
+    "PLANT_PDEMAND_COL": OrderedDict(
+        [
+            (
+                ("PLANT_PDEMAND_COL",),
+                lambda v: convert_units(v, target_units="mg*/m^2/day"),
+            )
+        ]
+    ),
+    "SMINN_TO_PLANT": OrderedDict(
+        [(("SMINN_TO_PLANT",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "SMINP_TO_PLANT": OrderedDict(
+        [(("SMINP_TO_PLANT",), lambda v: convert_units(v, target_units="mg*/m^2/day"))]
+    ),
+    "SMIN_NO3_LEACHED": OrderedDict(
+        [
+            (
+                ("SMIN_NO3_LEACHED",),
+                lambda v: convert_units(v, target_units="mg*/m^2/day"),
+            )
+        ]
+    ),
+    "FP_UPTAKE": OrderedDict(
+        [
+            (("FP_UPTAKE",), rename),
+            (
+                ("SMINN_TO_PLANT", "PLANT_NDEMAND_COL"),
+                lambda a, b: fp_uptake(a, b),
+            ),
+        ]
+    ),
     # Ocean variables
     "tauuo": OrderedDict([(("tauuo",), rename)]),
     "tos": OrderedDict([(("tos",), rename)]),

diff --git a/e3sm_diags/derivations/default_regions.py b/e3sm_diags/derivations/default_regions.py
@@ -15,6 +15,7 @@
     "50N90N": {"domain": cdutil.region.domain(latitude=(50.0, 90, "ccb"))},
     "60S90N": {"domain": cdutil.region.domain(latitude=(-60.0, 90, "ccb"))},
     "60S60N": {"domain": cdutil.region.domain(latitude=(-60.0, 60, "ccb"))},
+    "75S75N": {"domain": cdutil.region.domain(latitude=(-75.0, 75, "ccb"))},
     "ocean": {
         "value": 0.65,
     },