[develop] Simplify the way the configuration of the vx is handled

parm/metplus/EnsembleStat.conf

@@ -242,136 +242,123 @@ Import the file containing jinja macros.
 
 {#-
 Jinja requires certain variables to be defined globally within the template
-before they can be used in if-statements and other scopes (see Jinja
-scoping rules).  Define such variables.
+before they can be used in if-statements and other scopes (see Jinja scoping
+rules).  Define such variables.
 #}
+{%- set indx_level_fcst = '' %}
+{%- set indx_input_thresh_fcst = '' %}
+{%- set error_msg = '' %}
+{%- set opts_indent = '' %}
+{%- set opts_indent_len = '' %}
+
+{%- set field_fcst = '' %}
+{%- set field_obs = '' %}
 {%- set level_fcst = '' %}
 {%- set level_obs = '' %}
-{%- set indx_level_fcst = '' %}
+{%- set thresh_fcst = '' %}
+{%- set thresh_obs = '' %}
 
-{%- set valid_threshes_fcst = [] %}
-{%- set valid_threshes_obs = [] %}
 {%- set threshes_fcst = [] %}
 {%- set threshes_obs = [] %}
-{%- set indx_input_thresh_fcst = '' %}
-
-{%- set opts_indent = '' %}
-{%- set opts_indent_len = '' %}
-{%- set tmp = '' %}
-{%- set error_msg = '' %}
-{#-
-Make sure that the set of field groups for forecasts and observations
-are identical.
-#}
-{%- set fgs_fcst = vx_config_dict['fcst'].keys()|list %}
-{%- set fgs_obs = vx_config_dict['obs'].keys()|list %}
-{%- if (fgs_fcst != fgs_obs) %}
-  {%- set error_msg = '\n' ~
-'The list of valid field groups for forecasts (fgs_fcst) must be identical\n' ~
-'to that for observations (fgs_obs) but isn\'t:\n' ~
-'  fgs_fcst = ' ~ fgs_fcst ~ '\n' ~
-'  fgs_obs = ' ~ fgs_obs %}
-  {{metplus_macros.print_err_and_quit(error_msg)}}
-{%- endif %}
 
 {#-
-Extract the lists of forecast and observation dictionaries containing
-the valid fields, levels, and thresholds corresponding to the specified
-field group (input_field_group).  Note that it would be simpler to have
-these be just dictionaries in which the keys are the field names (instead
-of them being LISTS of dictionaries in which each dictionary contains a
-single key that is the field name), but that approach cannot be used here
-because it is possible for field names to be repeated (for both forecasts
-and observations).  For example, in the observations, the field name
-'PRWE' appears more than once, each time with a different threshold, and
-the combination of name and threshold is what constitutes a unique field,
-not just the name by itself.
+Get the set of valid field groups and ensure that the specified input
+field group appears in this list.
 #}
-{%- set fields_levels_threshes_fcst = vx_config_dict['fcst'][input_field_group] %}
-{%- set fields_levels_threshes_obs = vx_config_dict['obs'][input_field_group] %}
+{%- set valid_field_groups = vx_config_dict.keys()|list %}
+{{- metplus_macros.check_field_group(valid_field_groups, input_field_group) }}
 
 {#-
-Reset the specified forecast level so that if it happens to be an
-accumulation (e.g. 'A03'), the leading zeros in front of the hour are
-stipped out (e.g. reset to 'A3').
+Reset the input forecast level so that if it happens to be an accumulation
+(e.g. 'A03'), the leading zeros in front of the hour are stipped out (e.g.
+reset to 'A3').
 #}
 {%- set input_level_fcst = metplus_macros.get_accumulation_no_zero_pad(input_level_fcst) %}
 
 {#-
-Ensure that the specified input forecast level(s) (input_level_fcst) and
-threshold(s) (input_thresh_fcst) are valid, i.e. that they are in the
-set(s) of valid forecast levels and thresholds, respectively, specified
-in fields_levels_threshes_fcst.
+Extract from the configuration dictionary the set (which itself is a
+dictionary) of fields, levels, and thresholds corresponding to the input
+field group.  Then set the delimiter string that separates forecast and
+observation values in the various items (i.e. dictionary keys and values
+representing field names, levels, and thresholds) in this dictionary.
 #}
-{{- metplus_macros.check_level(fields_levels_threshes_fcst, input_level_fcst) }}
-{{- metplus_macros.check_thresh(fields_levels_threshes_fcst, input_level_fcst, input_thresh_fcst) }}
+{%- set fields_levels_threshes_cpld = vx_config_dict[input_field_group] %}
+{%- set delim_str = metplus_macros.set_delim_str() %}
 
 {#-
-For convenience, create lists of valid forecast and observation field
-names.
+Loop over the fields and set field names, levels, thresholds, and/or
+options for each field, both for forecasts and for observations, in the
+METplus configuration file.
 #}
-{%- set num_valid_fields_fcst = fields_levels_threshes_fcst|length %}
-{%- set valid_fields_fcst = [] %}
-{%- for i in range(0,num_valid_fields_fcst) %}
-  {%- set field = fields_levels_threshes_fcst[i].keys()|list|join('') %}
-  {%- set tmp = valid_fields_fcst.append(field) %}
-{%- endfor %}
+{%- set ns = namespace(var_count = 0) %}
+{%- for field_cpld, levels_threshes_cpld in fields_levels_threshes_cpld.items() %}
 
-{%- set valid_fields_obs = [] %}
-{%- set num_valid_fields_obs = fields_levels_threshes_obs|length %}
-{%- for i in range(0,num_valid_fields_obs) %}
-  {%- set field = fields_levels_threshes_obs[i].keys()|list|join('') %}
-  {%- set tmp = valid_fields_obs.append(field) %}
-{%- endfor %}
+  {%- if delim_str in field_cpld %}
+    {%- set field_fcst, field_obs = field_cpld.split(delim_str) %}
+  {%- else %}
+    {%- set field_fcst = field_cpld %}
+    {%- set field_obs = field_cpld %}
+  {%- endif %}
 
 {#-
-Ensure that the number of valid fields for forecasts is equal to that
-for the observations.
+For convenience, create lists of valid forecast and observation levels
+for the current field.
 #}
-{%- set num_valid_fields = 0 %}
-{%- if (num_valid_fields_fcst != num_valid_fields_obs) %}
-  {%- set error_msg = '\n' ~
-'The number of valid forecast fields (num_valid_fields_fcst) must be\n' ~
-'equal to the number of valid observation fields (num_valid_fields_obs)\n' ~
-'but isn\'t:\n' ~
-'  num_valid_fields_fcst = ' ~ num_valid_fields_fcst ~ '\n' ~
-'  num_valid_fields_obs = ' ~ num_valid_fields_obs ~ '\n' ~
-'The lists of valid forecast and observation fields are:\n' ~
-'  valid_fields_fcst = ' ~ valid_fields_fcst ~ '\n' ~
-'  valid_fields_obs = ' ~ valid_fields_obs ~ '\n' %}
-  {{metplus_macros.print_err_and_quit(error_msg)}}
-{%- else %}
-  {%- set num_valid_fields = num_valid_fields_fcst %}
-{%- endif %}
+  {%- set valid_levels_fcst = [] %}
+  {%- set valid_levels_obs = [] %}
+  {%- for level_cpld, threshes_cpld in levels_threshes_cpld.items() %}
+    {%- if delim_str in level_cpld %}
+      {%- set level_fcst, level_obs = level_cpld.split(delim_str) %}
+    {%- else %}
+      {%- set level_fcst = level_cpld %}
+      {%- set level_obs = level_cpld %}
+    {%- endif %}
+    {%- set tmp = valid_levels_fcst.append(level_fcst) %}
+    {%- set tmp = valid_levels_obs.append(level_obs) %}
+  {%- endfor %}
 
 {#-
-Loop over the valid fields and set field names, levels, thresholds, and/
-or options for each field, both for forecasts and for obseratiions, in
-the METplus configuration file.
+Make sure that the input forecast level (input_level_fcst) is set to a
+valid value.
 #}
-{%- set ns = namespace(var_count = 0) %}
-{%- for i in range(0,num_valid_fields) %}
-
-  {%- set field_fcst = valid_fields_fcst[i] %}
-  {%- set field_obs = valid_fields_obs[i] %}
+  {%- if (input_level_fcst != 'all') and (input_level_fcst not in valid_levels_fcst) %}
+    {%- set error_msg = '\n' ~
+'The input forecast level (input_level_fcst) must be set either to \'all\'\n' ~
+'or to one of the elements in the list of valid levels (valid_levels_fcst)\n' ~
+'for the current forecast field (field_fcst).  This is not the case:\n' ~
+'  field_fcst = ' ~ field_fcst ~ '\n' ~
+'  valid_levels_fcst = ' ~ valid_levels_fcst ~ '\n' ~
+'  input_level_fcst = ' ~ input_level_fcst ~ '\n' %}
+    {{metplus_macros.print_err_and_quit(error_msg)}}
+  {%- endif %}
 
 {#-
-For convenience, create lists of valid forecast and observation levels
-for the current field.  Then check that the number of valid levels for
-forecasts is the same as that for observations.
+Loop over the (coupled) levels and corresponding lists of thresholds.
+Extract from these the level values for forecasts and observations and
+use them to set the forecast and observation field names, levels,
+thresholds, and/or options in the METplus configuration file.
 #}
-  {%- set valid_levels_fcst = fields_levels_threshes_fcst[i][field_fcst].keys()|list %}
-  {%- set valid_levels_obs = fields_levels_threshes_obs[i][field_obs].keys()|list %}
+  {%- for level_cpld, threshes_cpld in levels_threshes_cpld.items() %}
 
-{#-
-Extract dictionary of valid forecast levels (the dictionary keys) and
-corresponding lists of valid thresholds (the values) for each level.
-Then loop over these levels and corresponding lists of thresholds to set
-both the forecast and observation field names, levels, thresholds, and/or
-options.
-#}
-  {%- set valid_levels_threshes_fcst = fields_levels_threshes_fcst[i][field_fcst] %}
-  {%- for level_fcst, valid_threshes_fcst in valid_levels_threshes_fcst.items() %}
+    {%- if delim_str in level_cpld %}
+      {%- set level_fcst, level_obs = level_cpld.split(delim_str) %}
+    {%- else %}
+      {%- set level_fcst = level_cpld %}
+      {%- set level_obs = level_cpld %}
+    {%- endif %}
+
+    {%- set valid_threshes_fcst = [] %}
+    {%- set valid_threshes_obs = [] %}
+    {%- for thresh_cpld in threshes_cpld %}
+      {%- if delim_str in thresh_cpld %}
+        {%- set thresh_fcst, thresh_obs = thresh_cpld.split(delim_str) %}
+      {%- else %}
+        {%- set thresh_fcst = thresh_cpld %}
+        {%- set thresh_obs = thresh_cpld %}
+      {%- endif %}
+      {%- set tmp = valid_threshes_fcst.append(thresh_fcst) %}
+      {%- set tmp = valid_threshes_obs.append(thresh_obs) %}
+    {%- endfor %}
 
     {%- if (input_level_fcst == 'all') or (input_level_fcst == level_fcst) %}
 {#-
@@ -415,17 +402,19 @@ to the full set of valid values.
           {%- set threshes_fcst = valid_threshes_fcst %}
 {#-
 If input_thresh_fcst is set to a specific value:
-  1) Ensure that input_thresh_fcst exists in the list of valid forecast
-     thresholds.
-  2) Get the index of input_thresh_fcst in the list of valid forecast
-     thresholds.  This will be needed later below when setting the
-     observation threshold(s).
-  3) Use this index to set the forecast threshold to a one-element list
-     containing the specified forecast threshold.
+* If that value is valid, i.e. it exists in the list of valid forecast
+  thresholds, get its index in that list and use it to set the forecast
+  threshold to a one-element list containing that value.  Note that the
+  index will be needed later below when setting the observation threshold(s).
+* If the input forecast threshold is not valid, print out a warning message
+  and exit.
 #}
         {%- else %}
 
-          {%- if input_thresh_fcst not in valid_threshes_fcst %}
+          {%- if input_thresh_fcst in valid_threshes_fcst %}
+            {%- set indx_input_thresh_fcst = valid_threshes_fcst.index(input_thresh_fcst) %}
+            {%- set threshes_fcst = [valid_threshes_fcst[indx_input_thresh_fcst]] %}
+          {%- else %}
             {%- set error_msg = '\n' ~
 'For the current forecast field (field_fcst) and forecast level (level_fcst),\n' ~
 'the input forecast threshold (input_thresh_fcst) does not exist in the list\n' ~
@@ -436,8 +425,6 @@ If input_thresh_fcst is set to a specific value:
 '  input_thresh_fcst = ' ~ input_thresh_fcst ~ '\n' %}
             {{metplus_macros.print_err_and_quit(error_msg)}}
           {%- endif %}
-          {%- set indx_input_thresh_fcst = valid_threshes_fcst.index(input_thresh_fcst) %}
-          {%- set threshes_fcst = [valid_threshes_fcst[indx_input_thresh_fcst]] %}
 
         {%- endif %}
 {#-
@@ -525,7 +512,7 @@ Set observation field name.  Note that this has to exactly match the name
 of the field in the input observation file.
 
 For accumulated fields, the input observation file is generated by MET's
-PcpCombine tool.  In that file, the field name consists of the observation 
+PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
@@ -557,11 +544,6 @@ set to 'none'.
 #}
       {%- if (input_thresh_fcst != 'none') %}
 {#-
-Set the list of valid observation thresholds to the one corresponding to
-the current observation level (level_obs).
-#}
-        {%- set valid_threshes_obs = fields_levels_threshes_obs[i][field_obs][level_obs] %}
-{#-
 If input_thresh_fcst is set to 'all', set the list of observation thresholds
 to the full set of valid values.
 #}
@@ -653,6 +635,7 @@ OBS_VAR{{ns.var_count}}_OPTIONS = desc = "TKE";
         {%- endif %}
 
       {%- endif %}
+
 {#-
 Print out a newline to separate the settings for the current field (both
 forecast and observation settings) from those for the next field.