Additional checks under section 3 for CF-1.7 (Ticket #29)

src/cfchecker/cfchecks.py

@@ -22,11 +22,14 @@
  -a or --area_types:
        the location of the CF area types table (xml)
        
+ -h or --help: Prints this help text
+
+ -r or --region_names:
+       the location of the CF standardized region names table (xml)
+
  -s or --cf_standard_names:
        the location of the CF standard name table (xml)
 
- -h or --help: Prints this help text.
-
  -v or --version: CF version to check against, use auto to auto-detect the file version.
 
 '''
@@ -42,13 +45,17 @@
 
 from netCDF4 import Dataset as netCDF4_Dataset
 from netCDF4 import Variable as netCDF4_Variable
+
 from cfunits import Units
 
+from operator import mul
+
 # Version is imported from the package module cfchecker/__init__.py
 from cfchecker import __version__
 
 STANDARDNAME = 'http://cfconventions.org/Data/cf-standard-names/current/src/cf-standard-name-table.xml'
 AREATYPES = 'http://cfconventions.org/Data/area-type-table/current/src/area-type-table.xml'
+REGIONNAMES = 'http://cfconventions.org/Data/cf-standard-names/docs/standardized-region-names.xml'
 
 #-----------------------------------------------------------
 from xml.sax import ContentHandler
@@ -285,13 +292,14 @@ class FatalCheckerError(Exception):
 #======================
 class CFChecker:
 
-  def __init__(self, uploader=None, useFileName="yes", badc=None, coards=None, cfStandardNamesXML=STANDARDNAME, cfAreaTypesXML=AREATYPES, version=newest_version, debug=False, silent=False):
+  def __init__(self, uploader=None, useFileName="yes", badc=None, coards=None, cfStandardNamesXML=STANDARDNAME, cfAreaTypesXML=AREATYPES, cfRegionNamesXML=REGIONNAMES, version=newest_version, debug=False, silent=False):
       self.uploader = uploader
       self.useFileName = useFileName
       self.badc = badc
       self.coards = coards
       self.standardNames = cfStandardNamesXML
       self.areaTypes = cfAreaTypesXML
+      self.regionNames = cfRegionNamesXML
       self.version = version
       self.all_results = OrderedDict()  # dictonary of results sorted by file and then by globals / variable 
                                         # and then by category
@@ -355,6 +363,11 @@ def checker(self, file):
         self.area_type_lh = ConstructList()
         parser.setContentHandler(self.area_type_lh)
         parser.parse(self.areaTypes)
+
+    # Set up list of valid region_names
+    self.region_name_lh = ConstructList()
+    parser.setContentHandler(self.region_name_lh)
+    parser.parse(self.regionNames)
 
     self._add_version("Using CF Checker Version %s" % __version__)
     if not self.version:
@@ -368,6 +381,9 @@ def checker(self, file):
     if self.version >= vn1_4:
         self._add_version("Using Area Type Table Version %s (%s)" % 
                        (self.area_type_lh.version_number, self.area_type_lh.last_modified))
+
+    self._add_version("Using Standardized Region Name Table Version %s (%s)" %
+                      (self.region_name_lh.version_number, self.region_name_lh.last_modified))
 
     if not self.silent:
         print ""
@@ -2196,8 +2212,46 @@ def chkDescription(self, varName):
                   modifier=std_name_el[1]
                   if not modifier in ['detection_minimum','number_of_observations','standard_error','status_flag']:
                       self._add_error("Invalid standard_name modifier: %s" % modifier, varName, code="3.3")
-
 
+                  if self.version >= vn1_7:
+                      if modifier in ['status_flag', 'number_of_observations']:
+                          self._add_warn("Use of standard_name modifier %s is deprecated" % modifier,
+                                         varName, code="3.3")
+
+              if name == "region":
+                  # Check values are from the permitted list
+                  region_names = self.getStringValue(varName)
+                  for region in region_names:
+                      if not region in self.region_name_lh.list:
+                          self._add_error("Invalid region name: %s" % region, varName, code="3.3")
+
+  #---------------------------------  
+  def getStringValue(self, varName):
+  #---------------------------------
+      # Collapse (by concatenation) the outermost
+      # (fastest varying) dimension of string valued array into
+      # memory. E.g. [['a','b','c']] becomes ['abc']
+      array=self.f.variables[varName][:]
+
+      if array.dtype.kind == 'S':
+          strlen = array.shape[-1]
+
+          new_shape = array.shape[0:-1]
+          new_size  = long(reduce(mul, new_shape, 1))
+
+          array = numpy.ma.resize(array, (new_size, strlen))
+
+          array = array.filled(fill_value='')
+
+          array = numpy.array([''.join(x).rstrip() for x in array],
+                              dtype='S%d' % strlen)
+
+          array = array.reshape(new_shape)
+
+          array = numpy.ma.where(array=='', numpy.ma.masked, array)
+
+      return array
+
   #-----------------------------------
   def chkCompressAttr(self, varName):
   #-----------------------------------
@@ -2452,9 +2506,11 @@ def getargs(arglist):
 
     standardnamekey='CF_STANDARD_NAMES'
     areatypeskey='CF_AREA_TYPES'
+    regionnameskey='CF_REGION_NAMES'
     # set defaults
     standardname=STANDARDNAME
     areatypes=AREATYPES
+    regionnames=REGIONNAMES
     uploader=None
     useFileName="yes"
     badc=None
@@ -2467,9 +2523,11 @@ def getargs(arglist):
         standardname=environ[standardnamekey]
     if environ.has_key(areatypeskey):
         areatypes=environ[areatypeskey]
+    if environ.has_key(regionnameskey):
+        regionnames=environ[regionnameskey]
 
     try:
-        (opts,args)=getopt(arglist[1:],'a:bcdhlns:v:',['area_types=','badc','coards','help','uploader','noname','cf_standard_names=','version=', 'debug'])
+        (opts,args)=getopt(arglist[1:],'a:bcdhlnr:s:v:',['area_types=','badc','coards','help','uploader','noname','region_names=','cf_standard_names=','version=', 'debug'])
     except GetoptError:
         stderr.write('%s\n'%__doc__)
         exit(1)
@@ -2496,6 +2554,9 @@ def getargs(arglist):
         if a in ('-n','--noname'):
             useFileName="no"
             continue
+        if a in ('-r','--region_names'):
+            regionnames=v.strip()
+            continue
         if a in ('-s','--cf_standard_names'):
             standardname=v.strip()
             continue

test_files/CF_1_0_OK.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CF_1_0_OK.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/CF_1_7.check

@@ -4,6 +4,7 @@ Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.7
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
 Using Area Type Table Version 6 (22 February 2017)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 ERROR: (2.6.3): Variable external_var2 named as an external variable must not be present in this file
 

test_files/CRM018_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM018_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (2.6.1): No 'Conventions' attribute present
 ERROR: (7.1): bounds attribute referencing non-existent variable bounds_lat

test_files/CRM021_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM021_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/CRM024_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM024_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 ERROR: (2.6.1): This netCDF file does not appear to contain CF Convention data.
 ERROR: (5.6): MSLP - Invalid syntax for 'grid_mapping' attribute

test_files/CRM026_test2.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM026_test2.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/CRM027_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM027_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (7.1): Data for variable time lies outside cell boundaries
 

test_files/CRM027_test2.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM027_test2.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (7.1): Data for variable time lies outside cell boundaries
 

test_files/CRM028_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM028_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (2.6.1): No 'Conventions' attribute present
 

test_files/CRM032_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM032_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 ERROR: (2.6.1): This netCDF file does not appear to contain CF Convention data.
 ERROR: (5.6): MSLP - Invalid syntax for 'grid_mapping' attribute

test_files/CRM033_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM033_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (2.6.1): No 'Conventions' attribute present
 

test_files/CRM035.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM035.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (2.6.1): No 'Conventions' attribute present
 

test_files/CRM037.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM037.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/CRM038.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM038.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/CRM041.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: CRM041.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/GregRappa.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: GregRappa.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/Trac020_test1.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: Trac020_test1.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/Trac020_test2.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: Trac020_test2.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/Trac022.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: Trac022.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (2.6.1): No 'Conventions' attribute present
 

test_files/Trac049_test1.check

@@ -4,6 +4,7 @@ Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.4
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
 Using Area Type Table Version 6 (22 February 2017)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/Trac049_test2.check

@@ -4,6 +4,7 @@ Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.4
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
 Using Area Type Table Version 6 (22 February 2017)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/cell_measures.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: cell_measures.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 ERROR: (7.1): Incorrect dimensions for boundary variable: lon_vertices
 WARN: (7.1): Data for variable lon lies outside cell boundaries

test_files/cell_methods.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: cell_methods.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 ERROR: (7.1): Incorrect dimensions for boundary variable: lon_vertices
 WARN: (7.1): Data for variable lon lies outside cell boundaries

test_files/complex.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: complex.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (2.6.1): No 'Conventions' attribute present
 WARN: (7.1): Data for variable lat lies outside cell boundaries

test_files/example_6.2.check

@@ -0,0 +1,29 @@
+CHECKING NetCDF FILE: example_6.2.nc
+=====================
+Using CF Checker Version 3.0.6-dev
+Checking against CF Version CF-1.7
+Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Area Type Table Version 6 (22 February 2017)
+Using Standardized Region Name Table Version 2 (12 June 2013)
+
+
+------------------
+Checking variable: n_heat_transport
+------------------
+
+------------------
+Checking variable: time
+------------------
+ERROR: (5): co-ordinate variable not monotonic
+
+------------------
+Checking variable: lat
+------------------
+
+------------------
+Checking variable: geo_region
+------------------
+
+ERRORS detected: 1
+WARNINGS given: 0
+INFORMATION messages: 0

test_files/flag_tests.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: flag_tests.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.3
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 
 ------------------

test_files/formula_terms.check

@@ -3,6 +3,7 @@ CHECKING NetCDF FILE: formula_terms.nc
 Using CF Checker Version 3.0.6-dev
 Checking against CF Version CF-1.0
 Using Standard Name Table Version 48 (2017-11-28T15:32:48Z)
+Using Standardized Region Name Table Version 2 (12 June 2013)
 
 WARN: (7.1): Data for variable lat lies outside cell boundaries