Add regression tests

Makefile

@@ -0,0 +1,14 @@
+SHELL = /bin/bash
+
+.PHONY: help test
+
+help: ## display help on Makefile targets
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+.DEFAULT_GOAL := help
+
+test: ## run tests
+	cd tests/regression; \
+	. env.sh; \
+	. env/bin/activate; \
+	pytest -v -s

checkers/diag_tolerance_check.py

@@ -14,7 +14,6 @@
 # private modules
 sys.path.append(os.path.join(os.path.dirname(__file__), "../tools"))
 from ts_utilities import read_environ, dir_path, str_to_bool
-from ts_fortran_nl import get_param
 from comp_table import cmp_table
 import ts_thresholds
 

checkers/existence_grib_out.sh

@@ -9,6 +9,15 @@
 # Email        [email protected]
 # Maintainer   [email protected]
 
+# this is needed for MacOS X
+find_one_file() {
+    if hash gfind 2>/dev/null; then
+        gfind "$@" | head -1
+    else
+        find "$@" | head -1
+    fi
+}
+
 # check environment variables
 RUNDIR=${TS_RUNDIR}
 VERBOSE=${TS_VERBOSE}
@@ -55,8 +64,16 @@ function grc {
 }
 
 # Regex pattern to match:
-REGEX=".+\/l[bf]ff0+\(_0\)?"
-FILE=$(find ${RUNDIR}/output -regex "${REGEX}") 
+REGEX="..*\/l[bf]ff0[0-9]*\(_0\)?"
+FILE=$(find_one_file ${RUNDIR}/output -regex "${REGEX}")
+
+# check if files exist
+if [ -z "${FILE}" ]; then
+  if [ "$VERBOSE" -gt 0 ]; then
+    echo "No grib output files found"
+  fi
+  exit 20 # FAIL
+fi
 
 # determine number of IO processors
 nprocio=0
@@ -74,13 +91,14 @@ if [ "${nprocio}" -gt 1 ] ; then
     cd ${cwd}
   fi
 fi
+
 # Search for the file again, because grc might have renamed the file
-FILE=$(find ${RUNDIR}/output -regex "${REGEX}") 
+FILE=$(find_one_file ${RUNDIR}/output -regex "${REGEX}")
 
 # check presence of output file
 if [ ! -s "${FILE}" ]; then
   if [ "$VERBOSE" -gt 0 ]; then
-    echo "File $FILE does not exists or is zero size"
+    echo "File $FILE does not exist or is zero size"
   fi
   exit 20 # FAIL
 fi

checkers/existence_ncdf_out.sh

@@ -24,7 +24,7 @@ if [ ! -d "${RUNDIR}" ] ; then
   exit 20 # FAIL
 fi
 
-FILELIST=$(ls -1 ${RUNDIR}/output/l[bf]f*00.nc 2>/dev/null)
+FILELIST=$(ls -1 ${RUNDIR}/output/l[bf]ff[0-9][0-9]*.nc 2>/dev/null)
 if [ $? -ne 0 ] ; then
   echo "No netCDF output file found"  1>&1
   exit 20 # FAIL

checkers/identical_check.py

@@ -49,7 +49,9 @@ def check():
     if switch: # extract timestep
         nlfile = switch  # namelist file containing dt
         try:
-            dt = float(get_param(rundir+nlfile, 'dt'))
+            dt = get_param(rundir+nlfile, 'dt')
+            assert(dt != '')
+            dt = float(dt)
         except:
             if verbose:
                 print header+'failed to extract dt from '+rundir+nlfile

checkers/output_tolerance_check.py

@@ -57,7 +57,9 @@ def check():
 
         # extract timestep
         try:
-            dt = float(get_param(rundir+nlfile2, 'dt'))
+            dt = get_param(rundir+nlfile2, 'dt')
+            assert(dt != '')
+            dt = float(dt)
         except:
             if verbose:
                 print header+'failed to extract dt from '+rundir+nlfile2
@@ -66,13 +68,17 @@ def check():
         # check for output lists
         try:
             nout_list = get_param(rundir+nlfile, lnout)
+            assert(nout_list != '')
+            nout_list = int(nout_list)
         except:
             if verbose:
                 print header+'no output lists in '+rundir+nlfile
             return 20 # FAIL
 
         # check if special testsuite output was activated in every output list
-        if get_param(rundir+nlfile, yuswitch, occurrence=nout_list) in ['.FALSE.', '.false.']:
+        yuswitch_value = get_param(rundir+nlfile, yuswitch, occurrence=nout_list)
+        assert(yuswitch_value != '')
+        if yuswitch_value in ['.FALSE.', '.false.']:
             if verbose:
                 print yuswitch+' is set to .false. in '+rundir+nlfile+' for this simulation'
             return 20 # FAIL

checkers/run_success_check.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 """
 COSMO TECHNICAL TESTSUITE
@@ -46,7 +46,7 @@ def run_checker():
             OccurrenceCrashPattern(     "Cleanup pattern",      "(.*)^(.*)CLEAN(\s*)UP(.*)" )
         ]
 
-    cosmo_filechecker = FileChecker()
+    cosmo_filechecker = FileChecker(verbose)
     cosmo_filechecker.add_pattern_list(patterns)
     return cosmo_filechecker.check(logfile, verbose)
 

checkers/tolerance_check.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 """
 COSMO TECHNICAL TESTSUITE
@@ -46,7 +46,7 @@ def check():
     icon = str_to_bool(env['ICON'])
     yufile = env['YUFILE']
 
-    #check if namelist file with switch exists in namelistdir
+    # check if namelist file with switch exists in namelistdir
     switch_path = namelistdir + switch
     if not os.path.exists(switch_path):
         if verbosity>0:
@@ -58,15 +58,16 @@ def check():
     yufile2 = refoutdir + yufile
     # check if special testsuite output was activated
     if not icon:
-       if get_param(rundir+switch, yuswitch) in ['.FALSE.', '.false.']:
-          if verbosity:
-             print yuswitch +' is set to .false. in '+ rundir + switch +' for this simulation'
-          return 20 # FAIL
-
-    #check if tolerance file exists in namelistdir or type dir
+        yuswitch_value = get_param(rundir+switch, yuswitch)
+        assert(yuswitch_value != '')
+        if yuswitch_value in ['.FALSE.', '.false.']:
+            if verbosity:
+               print yuswitch +' is set to .false. in '+ rundir + switch +' for this simulation'
+            return 20 # FAIL
+
+    # check if tolerance file exists in namelistdir or type dir
+    # Note: the forcematch option gives precedence over the tolerance file
     tolerance_path = namelistdir + tolerance
-
-    # The forcematch option gives precedence over the tolerance file
     if forcematch:
         thresh = """
         minval =    0.0
@@ -79,6 +80,14 @@ def check():
             print header + "unable to find tolerance file at " + tolerance
         return 20 # FAIL
 
+    if (verbosity > 1):
+        print header + "     yufile1: " + yufile1
+        print header + "     yufile2: " + yufile2
+        if forcematch:
+            print header + "  thresholds: " + "(force match)"
+        else:
+            print header + "  thresholds: " + tolerance_path
+
     try:
         c = Compare(yufile1, yufile2, thresh)
         if reset_thresholds:

tests/regression/.gitignore

@@ -0,0 +1,3 @@
+env
+.coverage
+htmlcov

tests/regression/clean

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+\rm -rf work *.out *.pyc __pycache__ .pytest_cache data/ .coverage htmlcov

tests/regression/config.cfg

@@ -0,0 +1,24 @@
+[ts_config]
+# List of INPUT namelist 
+# Should be a python list format
+l_files = ['INPUT_ORG','INPUT_ASS','INPUT_DIA','INPUT_DYN','INPUT_INI',\
+           'INPUT_IO','INPUT_PHY','INPUT_SAT','INPUT_POL','INPUT_IDEAL']
+# File containing the parallelisation option (nproc ...)
+par_file = INPUT_ORG
+# File containing time step information
+dt_file = INPUT_ORG
+# File containing lasync_io flag
+io_file = INPUT_IO
+# File containing test result
+res_file = TEST_RES
+# File used for the tolerance checker
+yufile = YUPRTEST
+# List of dual parameters (which may be interchanged in the namelists)
+# Should be a python list format
+dual_params=[('nstop','hstop'),('hincrad','nincrad')]
+# Where is the namelist switch for the testsuite output
+nl_ts_switch = INPUT_DIA
+# Where are all the namelists after execution
+config_nl = YUSPECIF
+# Is a perturbation necessary
+pert_avail = True

tests/regression/config2.cfg

@@ -0,0 +1 @@
+config.cfg

tests/regression/data_ref/basic/in_aux/aux_dummy

@@ -0,0 +1 @@
+This is a dummy aux file.

tests/regression/data_ref/basic/input/in_dummy

@@ -0,0 +1 @@
+This is a dummy input file.

tests/regression/data_ref/basic/test_basic/INPUT_DIA

@@ -0,0 +1,4 @@
+&diactl
+  ltestsuite=.TRUE.,
+/END
+

tests/regression/data_ref/basic/test_basic/INPUT_IO

@@ -0,0 +1,6 @@
+&ioctl
+  ngribout=1,
+/
+&gribout
+  lcheck=.true.,
+/

tests/regression/data_ref/basic/test_basic/INPUT_ORG

@@ -0,0 +1,9 @@
+&runctl
+  dt=60.0,
+  hstart=0.0,
+  hstop=2.0,
+  nprocx=1,
+  nprocy=1,
+  nprocio=0,
+  itype_pert=0,
+/

tests/regression/data_ref/basic/test_basic/TOLERANCE

@@ -0,0 +1,7 @@
+   minval = 1e-12
+    steps =          0          1          2          3          4          5          7         10         20         40        100
+        * =   0.00e+00   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   4.00e-11
+      eta =   0.00e+00   7.00e-15   7.00e-15   7.00e-15   7.00e-15   7.00e-15   7.00e-15   7.00e-15   9.00e-14   9.00e-14   2.00e-13
+        u =   0.00e+00   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   2.00e-13   4.00e-11
+        v =   0.00e+00   4.00e-14   4.00e-14   4.00e-14   4.00e-14   4.00e-14   4.00e-14   4.00e-14   4.00e-14   4.00e-14   4.00e-14
+   CHKDAT =   5.00e-05   1.00e-04   1.00e-01   1.00e+00   1.00e+00   1.00e+00   5.00e-05   1.00e-04   1.00e-01   1.00e+00   1.00e+00

tests/regression/data_ref/basic/test_basic/TOLERANCE2

@@ -0,0 +1 @@
+TOLERANCE

tests/regression/data_ref/basic/test_basic/YUCHKDAT

@@ -0,0 +1,10 @@
+
+Check the file data: 
+    File:   output/lfff00000120
+    ie_tot =   70   je_tot =    7   ke_tot =   80
+
+     var       ee    lev         min      imin   jmin          max      imax   jmax         mean  
+  u           13   1      -0.095288    0    0             0.003686    0    0            -0.014846
+  v           17   1      -0.113997    0    0             0.042260    0    0            -0.005102
+  eta         33   1      -0.169840    0    0             0.820299    0    0             0.055224
+  KE         102   1       0.000000    0    0             0.013075    0    0             0.001299

tests/regression/data_ref/basic/test_basic/YUPRTEST

@@ -0,0 +1,70 @@
+#    Experiment:    Model
+#    ie_tot =   49   je_tot =   49   ke =   63
+#
+#    var    nt  lev                         min imin jmin                         max imax jmax                        mean
+       u     0    1    0.000000000000000000E+00    0    0    0.000000000000000000E+00    0    0    0.000000000000000000E+00
+       v     0    1    0.000000000000000000E+00    0    0    0.000000000000000000E+00    0    0    0.000000000000000000E+00
+     eta     0    1    2.165689130516408471E-29    0    0    9.961278882600469453E-01    0    0    5.522382910304456799E-02
+       u     1    1   -3.849458898343552805E-03    0    0    3.857983612378607183E-03    0    0   -6.611470845315728807E-05
+       v     1    1   -3.835380936624159954E-03    0    0    3.849516122251034258E-03    0    0   -6.215758017089209848E-06
+     eta     1    1    2.345695763539030966E-29    0    0    9.953017139538279379E-01    0    0    5.522382910304456799E-02
+       u     2    1   -7.692658883466689350E-03    0    0    7.713223876914587773E-03    0    0   -1.324317283852256626E-04
+       v     2    1   -7.667989748219322492E-03    0    0    7.691689663357714063E-03    0    0   -1.225612305308798210E-05
+     eta     2    1    2.718304073672345290E-29    0    0    9.936507572269372979E-01    0    0    5.522382910304456799E-02
+       u     3    1   -1.152494372395403838E-02    0    0    1.156051196185309965E-02    0    0   -1.991318837089660722E-04
+       v     3    1   -1.149325898728429350E-02    0    0    1.152160487686155511E-02    0    0   -1.815950560166123610E-05
+     eta     3    1    3.309580697700215026E-29    0    0    9.911778070194778856E-01    0    0    5.522382910304458187E-02
+       u     4    1   -1.534167465409049837E-02    0    0    1.539399269279500583E-02    0    0   -2.663925937896652850E-04
+       v     4    1   -1.530663479479465149E-02    0    0    1.533436765403836666E-02    0    0   -2.396332646479783308E-05
+     eta     4    1    4.160699553335569057E-29    0    0    9.878870420813891107E-01    0    0    5.522382910304456799E-02
+       u     5    1   -1.913823830891700842E-02    0    0    1.920687789955326241E-02    0    0   -3.343861984290755728E-04
+       v     5    1   -1.910358449007975093E-02    0    0    1.912511457882534044E-02    0    0   -2.970382931367051131E-05
+     eta     5    1    5.330521103146963194E-29    0    0    9.837840200858807282E-01    0    0    5.522382910304456799E-02
+       u     6    1   -2.291005461310782407E-02    0    0    2.299120008110080646E-02    0    0   -4.032783810037476185E-04
+       v     6    1   -2.287960411729544896E-02    0    0    2.288902191552621632E-02    0    0   -3.541591222889588209E-05
+     eta     6    1    6.899267108538228284E-29    0    0    9.788756631372773631E-01    0    0    5.522382910304458187E-02
+       u     7    1   -2.665258460817618302E-02    0    0    2.673761756016777499E-02    0    0   -4.732267188548649963E-04
+       v     7    1   -2.663022593451126740E-02    0    0    2.662131450880340575E-02    0    0   -4.113298187172539634E-05
+     eta     7    1    8.973506632645738358E-29    0    0    9.731702397219794687E-01    0    0    5.522382910304456799E-02
+       u     8    1   -3.036133820253821350E-02    0    0    3.043528043192210084E-02    0    0   -5.443793868587664840E-04
+       v     8    1   -3.035102583148155397E-02    0    0    3.031727457561364872E-02    0    0   -4.688683339675258727E-05
+     eta     8    1    1.169274480257004141E-28    0    0    9.666773431636760394E-01    0    0    5.522382910304458187E-02
+       u     9    1   -3.403188182900170056E-02    0    0    3.407176440437785503E-02    0    0   -6.168740363169907589E-04
+       v     9    1   -3.403763066223040173E-02    0    0    3.397225036893154748E-02    0    0   -5.270755859825849830E-05
+     eta     9    1    1.523799762389538719E-28    0    0    9.594078666563252700E-01    0    0    5.522382910304456799E-02
+       u    10    1   -3.765984599430126367E-02    0    0    3.763307716062441743E-02    0    0   -6.908368669996921025E-04
+       v    10    1   -3.768572547864499495E-02    0    0    3.758166469333560494E-02    0    0   -5.862348410336022702E-05
+     eta    10    1    1.984284891171441316E-28    0    0    9.513739749608354224E-01    0    0    5.522382910304456799E-02
+       u    20    1   -7.069654193677141207E-02    0    0    6.660346746990539191E-02    0    0   -1.529986316873608405E-03
+       v    20    1   -7.115477628695109080E-02    0    0    7.034133622904563399E-02    0    0   -1.280182359828690487E-04
+     eta    20    1    2.578246346016506010E-27    0    0    8.325055673301089421E-01    0    0    5.522382910304455411E-02
+       u    30    1   -9.541409038619252936E-02    0    0    8.309606731653416956E-02    0    0   -2.567306370223154319E-03
+       v    30    1   -9.671195395352308322E-02    0    0    9.475093398228344455E-02    0    0   -2.272689720382698960E-04
+     eta    30    1    2.987407885692794554E-26    0    0    6.584545930077215781E-01    0    0    5.522382910304455411E-02
+       u    40    1   -1.100508530814234553E-01    0    0    7.719553232820777766E-02    0    0   -3.782634643842510552E-03
+       v    40    1   -1.119440132384657838E-01    0    0    1.106122350742410054E-01    0    0   -3.707919296440171297E-04
+     eta    40    1    3.181875335092288452E-25    0    0    6.659314870486774396E-01    0    0    5.522382910304454717E-02
+       u    50    1   -1.160769113112332385E-01    0    0    5.913819249444620535E-02    0    0   -5.148142292701780864E-03
+       v    50    1   -1.177561899680515944E-01    0    0    1.158975975538960868E-01    0    0   -5.782977354372791871E-04
+     eta    50    1    3.153598733213607455E-24    0    0    6.499141669012825639E-01    0    0    5.522382910304454717E-02
+       u    60    1   -1.133924228931027650E-01    0    0    3.604013761737683008E-02    0    0   -6.628222144932366909E-03
+       v    60    1   -1.162515148477110405E-01    0    0    1.125574336607608333E-01    0    0   -8.724761947243736288E-04
+     eta    60    1    2.925378209752125347E-23    0    0    5.993805666152879352E-01    0    0    5.522382910304454717E-02
+       u    70    1   -1.069155152024322680E-01    0    0    1.916776659423633650E-02    0    0   -8.166774150834121143E-03
+       v    70    1   -1.216608228562506000E-01    0    0    1.218252845732289841E-01    0    0   -1.273797721918859017E-03
+     eta    70    1   -6.665379545771726866E-02    0    0    5.499291077862874522E-01    0    0    5.522382910304451942E-02
+       u    80    1   -9.938236557169073671E-02    0    0    1.014516363878045432E-02    0    0   -9.703460325732330555E-03
+       v    80    1   -1.258966613369230658E-01    0    0    1.219229518559721004E-01    0    0   -1.798508543845048300E-03
+     eta    80    1   -1.465114359457190263E-01    0    0    5.556185440210257287E-01    0    0    5.522382910304451248E-02
+       u    90    1   -9.275845828397710768E-02    0    0    1.496996502908658298E-03    0    0   -1.118338017485132931E-02
+       v    90    1   -1.261216544495663561E-01    0    0    1.130126090780570053E-01    0    0   -2.454872303062005916E-03
+     eta    90    1   -1.765983184052934141E-01    0    0    6.322587299764972357E-01    0    0    5.522382910304451942E-02
+       u   100    1   -9.012746608338956245E-02    0    0    5.852564112425879790E-05    0    0   -1.255587669827768794E-02
+       v   100    1   -1.234004314201772035E-01    0    0    9.676074828756117685E-02    0    0   -3.238566829864447574E-03
+     eta   100    1   -1.778268219325232491E-01    0    0    7.293143756723945437E-01    0    0    5.522382910304453330E-02
+       u   110    1   -9.546339188624054783E-02    0    0    1.211407134026049710E-04    0    0   -1.378330988624681432E-02
+       v   110    1   -1.189629942758403808E-01    0    0    7.049313933752406058E-02    0    0   -4.131445647557008966E-03
+     eta   110    1   -1.608210421961591419E-01    0    0    7.962781502016331725E-01    0    0    5.522382910304451942E-02
+       u   120    1   -9.528805440797019632E-02    0    0    3.686192315303329480E-03    0    0   -1.484643587868229343E-02
+       v   120    1   -1.139968580611173399E-01    0    0    4.225988463019059865E-02    0    0   -5.102295963162439309E-03
+     eta   120    1   -1.698400505108406400E-01    0    0    8.202991184514930589E-01    0    0    5.522382910304451248E-02

tests/regression/data_ref/basic/test_basic/YUSPECIF

@@ -0,0 +1,12 @@
+
+0     The NAMELIST variables were specified as follows:
+      =================================================
+
+
+0     NAMELIST:  runctl
+      -----------------
+
+      Variable                  Actual Value      Default Value      Format
+       nprocx                              4                  0        I 
+       nprocy                              4                  0        I 
+       nprocio                             0                  0        I