diff --git a/setup.py b/setup.py index 8aed7b8..8021b20 100644 --- a/setup.py +++ b/setup.py @@ -39,4 +39,5 @@ entry_points= { 'console_scripts': ['cfchecks = cfchecker.cfchecks:main'], }, + scripts=['src/cf-checker'] ) diff --git a/src/cfchecker/cfchecks.py b/src/cfchecker/cfchecks.py index c99258e..4f0fdde 100644 --- a/src/cfchecker/cfchecks.py +++ b/src/cfchecker/cfchecks.py @@ -46,6 +46,7 @@ from netCDF4 import Dataset as netCDF4_Dataset from netCDF4 import Variable as netCDF4_Variable +from netCDF4 import VLType as netCDF4_VLType from cfunits import Units @@ -580,6 +581,16 @@ def _checker(self): self._add_debug("Axes: %s" % axes) + valid_types=[numpy.character, + numpy.dtype('c'), + numpy.dtype('b'), + numpy.dtype('i4'), + numpy.int32, + numpy.float32, + numpy.double, + 'int16', + 'float32'] + # Check each variable for var in self.f.variables.keys(): @@ -593,6 +604,14 @@ def _checker(self): if not self.validName(var): self._add_error("Invalid variable name", var, code='2.3') + dt = self.f.variables[var].dtype + if dt not in valid_types: + try: + if isinstance(self.f.variables[var].datatype, netCDF4_VLType): + self._add_error("Invalid variable type: {} (vlen types not supported)".format(self.f.variables[var].datatype), var, code="2.2") + except: + self._add_error("Invalid variable type: {}".format(dt), var, code="2.2") + # Check to see if a variable with this name already exists (case-insensitive) lowerVar=var.lower() if lowerVar in lowerVars: @@ -1023,7 +1042,7 @@ def getCoordinateDataVars(self): # self._add_warn("Data for variable %s lies outside cell boundaries" % var, # var, code="7.1") # else: - for i, value in enumerate(varData): + for i, value in (enumerate(varData) if len(varData.shape) else enumerate([varData])): try: if not (boundsData[i][0] <= value <= boundsData[i][1]): self._add_warn("Data for variable %s lies outside cell boundaries" % var, @@ -1234,7 +1253,7 @@ def chkGridMappingVar(self, varName): attr_type='N' else: - self._add_info("Unknown Type for attribute: %s %s" % (attribute, attr_type)) + self._add_info("Invalid Type for attribute: %s %s" % (attribute, attr_type)) continue if (attribute in self.grid_mapping_attrs.keys() and @@ -1623,8 +1642,22 @@ def getTypeCode(self, obj): # return obj.dtype.char # print "RSH: type ", obj.dtype.char - - return obj.dtype.char + + if isinstance(obj, netCDF4_Variable): + # Variable object + if isinstance(obj.datatype, netCDF4_VLType): + # VLEN types not supported + return 'vlen' + + try: + typecode = obj.dtype.char + except AttributeError as e: + self._add_warn("Problem getting typecode: {}".format(e), obj.name) + else: + # Attribute object + typecode = obj.dtype.char + + return typecode #------------------------------------------------------- @@ -1640,7 +1673,14 @@ def chkAttribute(self, attribute,varName,allCoordVars): varName) return - value=var.getncattr(attribute) + try: + value=var.getncattr(attribute) + except KeyError as e: + self._add_error("{} - {}".format(attribute,e), varName, code="2.2") + if self.AttrList.has_key(attribute): + # This is a standard attribute so inform user no further checks being made on it + self._add_info("No further checks made on attribute: {}".format(attribute), varName) + return self._add_debug("chkAttribute: Checking attribute - %s" % attribute, varName) @@ -1661,7 +1701,7 @@ def chkAttribute(self, attribute,varName,allCoordVars): elif attrType == types.NoneType: attrType='NoneType' else: - self._add_info("Unknown Type for attribute: %s %s" % (attribute, attrType)) + self._add_info("Invalid Type for attribute: %s %s" % (attribute, attrType)) # If attrType = 'NoneType' then it has been automatically created e.g. missing_value typeError=0 @@ -2183,12 +2223,15 @@ def chkUnits(self,varName,allCoordVars): #print "RSH: in allCoordVars" # Label variables do not require units attribute - if self.f.variables[varName].dtype.char != 'S': - if hasattr(var, 'axis'): - if not var.axis == 'Z': + try: + if self.f.variables[varName].dtype.char != 'S': + if hasattr(var, 'axis'): + if not var.axis == 'Z': + self._add_warn("units attribute should be present", varName, code="3.1") + elif not hasattr(var,'positive') and not hasattr(var,'formula_terms') and not hasattr(var,'compress'): self._add_warn("units attribute should be present", varName, code="3.1") - elif not hasattr(var,'positive') and not hasattr(var,'formula_terms') and not hasattr(var,'compress'): - self._add_warn("units attribute should be present", varName, code="3.1") + except: + pass elif varName not in self.boundsVars and varName not in self.climatologyVars and varName not in self.gridMappingVars: # Variable is not a boundary or climatology variable @@ -2653,7 +2696,7 @@ def chkFlags(self, varName): varName, code="3.5") # flag_values values must be mutually exclusive - if type(values) == str: + if isinstance(values, basestring): values = values.split() if not self.uniqueList(values): diff --git a/test_files/issue27.check b/test_files/issue27.check new file mode 100644 index 0000000..528b499 --- /dev/null +++ b/test_files/issue27.check @@ -0,0 +1,707 @@ +CHECKING NetCDF FILE: issue27.nc +===================== +Using CF Checker Version 3.0.6-dev +Checking against CF Version CF-1.6 +Using Standard Name Table Version 49 (2018-02-13T08:44:33Z) +Using Area Type Table Version 6 (22 February 2017) +Using Standardized Region Name Table Version 2 (12 June 2013) + +ERROR: (5): coordinates attribute referencing non-existent variable +ERROR: (5.3): Invalid syntax for 'coordinates' attribute + +------------------ +Checking variable: obs_id +------------------ +ERROR: (2.2): Invalid variable type: : string type (vlen types not supported) + +------------------ +Checking variable: fov_obs_id +------------------ +ERROR: (2.2): Invalid variable type: : string type (vlen types not supported) + +------------------ +Checking variable: obs_time_tai93 +------------------ + +------------------ +Checking variable: obs_time_utc +------------------ +INFO: Invalid Type for attribute: _FillValue + +------------------ +Checking variable: lat +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lat_geoid +------------------ + +------------------ +Checking variable: fov_lat +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lon +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lon_geoid +------------------ + +------------------ +Checking variable: fov_lon +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lat_bnds +------------------ +WARN: (7.1): Boundary Variable lat_bnds should not have _FillValue attribute + +------------------ +Checking variable: fov_lat_bnds +------------------ +WARN: (7.1): Boundary Variable fov_lat_bnds should not have _FillValue attribute + +------------------ +Checking variable: lon_bnds +------------------ +WARN: (7.1): Boundary Variable lon_bnds should not have _FillValue attribute + +------------------ +Checking variable: fov_lon_bnds +------------------ +WARN: (7.1): Boundary Variable fov_lon_bnds should not have _FillValue attribute + +------------------ +Checking variable: land_frac +------------------ + +------------------ +Checking variable: fov_land_frac +------------------ + +------------------ +Checking variable: surf_alt +------------------ + +------------------ +Checking variable: fov_surf_alt +------------------ + +------------------ +Checking variable: surf_alt_sdev +------------------ + +------------------ +Checking variable: fov_surf_alt_sdev +------------------ + +------------------ +Checking variable: sun_glint_lat +------------------ + +------------------ +Checking variable: sun_glint_lon +------------------ + +------------------ +Checking variable: sol_zen +------------------ + +------------------ +Checking variable: sol_azi +------------------ + +------------------ +Checking variable: sun_glint_dist +------------------ + +------------------ +Checking variable: view_ang +------------------ + +------------------ +Checking variable: sat_zen +------------------ + +------------------ +Checking variable: sat_azi +------------------ + +------------------ +Checking variable: sat_range +------------------ + +------------------ +Checking variable: asc_flag +------------------ +INFO: Invalid Type for attribute: _FillValue + +------------------ +Checking variable: subsat_lat +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: subsat_lon +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: scan_mid_time +------------------ + +------------------ +Checking variable: sat_alt +------------------ + +------------------ +Checking variable: sat_pos +------------------ + +------------------ +Checking variable: sat_vel +------------------ + +------------------ +Checking variable: sat_att +------------------ + +------------------ +Checking variable: local_solar_time +------------------ + +------------------ +Checking variable: mean_anom_wrt_equat +------------------ + +------------------ +Checking variable: sat_sol_zen +------------------ + +------------------ +Checking variable: sat_sol_azi +------------------ + +------------------ +Checking variable: asc_node_lon +------------------ + +------------------ +Checking variable: asc_node_tai93 +------------------ + +------------------ +Checking variable: asc_node_local_solar_time +------------------ + +------------------ +Checking variable: solar_beta_angle +------------------ + +------------------ +Checking variable: attitude_lbl +------------------ +ERROR: (2.2): Invalid variable type: : string type (vlen types not supported) +WARN: (2.2): Could not get typecode of variable. Variable types supported are: char, byte, short, int, float, real, double + +------------------ +Checking variable: spatial_lbl +------------------ +ERROR: (2.2): Invalid variable type: : string type (vlen types not supported) + +------------------ +Checking variable: utc_tuple_lbl +------------------ +ERROR: (2.2): Invalid variable type: : string type (vlen types not supported) + +------------------ +Checking variable: air_temp +------------------ + +------------------ +Checking variable: air_temp_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: air_temp_err +------------------ + +------------------ +Checking variable: surf_air_temp +------------------ + +------------------ +Checking variable: surf_air_temp_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: surf_air_temp_err +------------------ + +------------------ +Checking variable: air_temp_dof +------------------ + +------------------ +Checking variable: h2o_vap_tot +------------------ + +------------------ +Checking variable: h2o_vap_tot_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: h2o_vap_tot_err +------------------ + +------------------ +Checking variable: h2o_mmr +------------------ + +------------------ +Checking variable: h2o_mmr_qc +------------------ + +------------------ +Checking variable: h2o_mmr_err +------------------ + +------------------ +Checking variable: surf_h2o_mmr +------------------ + +------------------ +Checking variable: surf_h2o_mmr_qc +------------------ + +------------------ +Checking variable: surf_h2o_mmr_err +------------------ + +------------------ +Checking variable: h2o_vap_dof +------------------ + +------------------ +Checking variable: rel_hum +------------------ + +------------------ +Checking variable: rel_hum_qc +------------------ + +------------------ +Checking variable: rel_hum_err +------------------ + +------------------ +Checking variable: surf_rel_hum +------------------ + +------------------ +Checking variable: surf_rel_hum_qc +------------------ + +------------------ +Checking variable: surf_rel_hum_err +------------------ + +------------------ +Checking variable: h2o_mmr_sat_ice +------------------ +ERROR: (2.2): AIRS_name - 'vlen string array attributes not supported' + +------------------ +Checking variable: h2o_mmr_sat_ice_qc +------------------ + +------------------ +Checking variable: h2o_mmr_sat_ice_err +------------------ + +------------------ +Checking variable: surf_h2o_mmr_sat_ice +------------------ +ERROR: (2.2): AIRS_name - 'vlen string array attributes not supported' + +------------------ +Checking variable: surf_h2o_mmr_sat_ice_qc +------------------ + +------------------ +Checking variable: surf_h2o_mmr_sat_ice_err +------------------ + +------------------ +Checking variable: h2o_mmr_sat_liq +------------------ + +------------------ +Checking variable: h2o_mmr_sat_liq_qc +------------------ + +------------------ +Checking variable: h2o_mmr_sat_liq_err +------------------ + +------------------ +Checking variable: surf_h2o_mmr_sat_liq +------------------ + +------------------ +Checking variable: surf_h2o_mmr_sat_liq_qc +------------------ + +------------------ +Checking variable: surf_h2o_mmr_sat_liq_err +------------------ + +------------------ +Checking variable: gp_hgt +------------------ +INFO: attribute axis is being used in a non-standard way + +------------------ +Checking variable: gp_hgt_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: gp_hgt_err +------------------ + +------------------ +Checking variable: surf_gp_hgt +------------------ + +------------------ +Checking variable: surf_gp_hgt_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: surf_gp_hgt_err +------------------ + +------------------ +Checking variable: o3_tot +------------------ + +------------------ +Checking variable: o3_tot_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: o3_tot_err +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: o3_vmr +------------------ + +------------------ +Checking variable: o3_vmr_qc +------------------ + +------------------ +Checking variable: o3_vmr_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: o3_dof +------------------ + +------------------ +Checking variable: co_vmr_midtrop +------------------ + +------------------ +Checking variable: co_vmr_midtrop_qc +------------------ + +------------------ +Checking variable: co_vmr_midtrop_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: co_dof +------------------ + +------------------ +Checking variable: ch4_tot +------------------ + +------------------ +Checking variable: ch4_tot_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: ch4_tot_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: ch4_vmr +------------------ + +------------------ +Checking variable: ch4_vmr_qc +------------------ + +------------------ +Checking variable: ch4_vmr_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: ch4_dof +------------------ + +------------------ +Checking variable: mw_cld_phase +------------------ + +------------------ +Checking variable: h2o_liq_tot +------------------ + +------------------ +Checking variable: h2o_liq_tot_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: h2o_liq_tot_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: h2o_liq_mol_lay +------------------ + +------------------ +Checking variable: h2o_liq_mol_lay_qc +------------------ + +------------------ +Checking variable: h2o_liq_mol_lay_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: surf_temp +------------------ + +------------------ +Checking variable: surf_temp_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: surf_temp_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: surf_temp_dof +------------------ + +------------------ +Checking variable: surf_emis_ir +------------------ + +------------------ +Checking variable: surf_emis_ir_qc +------------------ + +------------------ +Checking variable: surf_emis_ir_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: surf_refl_ir +------------------ + +------------------ +Checking variable: surf_refl_ir_qc +------------------ + +------------------ +Checking variable: surf_emis_ir_freq_cnt +------------------ + +------------------ +Checking variable: surf_emis_ir_freq +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: surf_emis_mw +------------------ + +------------------ +Checking variable: surf_emis_mw_qc +------------------ + +------------------ +Checking variable: surf_emis_mw_err +------------------ +ERROR: (3.3): Invalid standard_name modifier: error_estimate + +------------------ +Checking variable: cld_frac_tot +------------------ + +------------------ +Checking variable: cld_frac +------------------ + +------------------ +Checking variable: cld_frac_qc +------------------ + +------------------ +Checking variable: cld_top_pres +------------------ + +------------------ +Checking variable: cld_top_pres_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: cld_top_temp +------------------ + +------------------ +Checking variable: cld_top_temp_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: olr +------------------ + +------------------ +Checking variable: olr_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: olr_band +------------------ + +------------------ +Checking variable: olr_clr +------------------ + +------------------ +Checking variable: olr_clr_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: olr_clr_band +------------------ + +------------------ +Checking variable: tpause_gp_hgt +------------------ + +------------------ +Checking variable: tpause_gp_hgt_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: tpause_pres +------------------ + +------------------ +Checking variable: tpause_pres_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: tpause_temp +------------------ + +------------------ +Checking variable: tpause_temp_qc +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: bndry_lyr_top +------------------ + +------------------ +Checking variable: bndry_lyr_top_qc +------------------ + +------------------ +Checking variable: surf_indx +------------------ + +------------------ +Checking variable: lev_100 +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lev_h2o +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lay_100 +------------------ +INFO: attribute _FillValue is being used in a non-standard way + +------------------ +Checking variable: lay_100_bnds +------------------ +WARN: (7.1): Boundary Variable lay_100_bnds should not have _FillValue attribute +ERROR: (7.3): Invalid syntax for cell_methods attribute + +------------------ +Checking variable: olr_freq +------------------ +INFO: attribute _FillValue is being used in a non-standard way +ERROR: (3.1): Units are not consistent with those given in the standard_name table. + +------------------ +Checking variable: olr_freq_bnds +------------------ +ERROR: (3.1): Units are not consistent with those given in the standard_name table. +WARN: (7.1): Boundary Variable olr_freq_bnds should not have _FillValue attribute + +------------------ +Checking variable: mw_surf_class +------------------ + +------------------ +Checking variable: surf_freq_mw +------------------ +INFO: attribute _FillValue is being used in a non-standard way +WARN: (4.4.1): Use of the calendar and/or month_lengths attributes is recommended for time coordinate variables +ERROR: (4.4): Invalid units and/or reference time + +ERRORS detected: 41 +WARNINGS given: 8 +INFORMATION messages: 14 diff --git a/test_files/issue27.nc.gz b/test_files/issue27.nc.gz new file mode 100644 index 0000000..b9f7f83 Binary files /dev/null and b/test_files/issue27.nc.gz differ diff --git a/test_files/tests.sh b/test_files/tests.sh index d376dbd..13890ea 100755 --- a/test_files/tests.sh +++ b/test_files/tests.sh @@ -38,7 +38,7 @@ do then # CF-1.4 $cfchecker -s $std_name_table -a $area_table -v 1.4 $file > $outdir/$file.out 2>&1 - elif [[ $file == "CF_1_7.nc" || $file == "example_6.2.nc" || $file == "example_5.10.nc" ]] + elif [[ $file == "CF_1_7.nc" || $file == "example_6.2.nc" || $file == "example_5.10.nc" || $file = "issue27.nc" ]] then # Run checker using the CF version specified in the conventions attribute of the file $cfchecker -s $std_name_table -v auto $file > $outdir/$file.out 2>&1