Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add regression tests #29

Merged
merged 15 commits into from
Nov 28, 2019
14 changes: 14 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
SHELL = /bin/bash

.PHONY: help test

help: ## display help on Makefile targets
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

.DEFAULT_GOAL := help

test: ## run tests
cd tests/regression; \
. env.sh; \
. env/bin/activate; \
pytest -v -s
1 change: 0 additions & 1 deletion checkers/diag_tolerance_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
# private modules
sys.path.append(os.path.join(os.path.dirname(__file__), "../tools"))
from ts_utilities import read_environ, dir_path, str_to_bool
from ts_fortran_nl import get_param
from comp_table import cmp_table
import ts_thresholds

Expand Down
26 changes: 22 additions & 4 deletions checkers/existence_grib_out.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@
# Email [email protected]
# Maintainer [email protected]

# this is needed for MacOS X
find_one_file() {
if hash gfind 2>/dev/null; then
gfind "$@" | head -1
else
find "$@" | head -1
fi
}

# check environment variables
RUNDIR=${TS_RUNDIR}
VERBOSE=${TS_VERBOSE}
Expand Down Expand Up @@ -55,8 +64,16 @@ function grc {
}

# Regex pattern to match:
REGEX=".+\/l[bf]ff0+\(_0\)?"
FILE=$(find ${RUNDIR}/output -regex "${REGEX}")
REGEX="..*\/l[bf]ff0[0-9]*\(_0\)?"
FILE=$(find_one_file ${RUNDIR}/output -regex "${REGEX}")

# check if files exist
if [ -z "${FILE}" ]; then
if [ "$VERBOSE" -gt 0 ]; then
echo "No grib output files found"
fi
exit 20 # FAIL
fi

# determine number of IO processors
nprocio=0
Expand All @@ -74,13 +91,14 @@ if [ "${nprocio}" -gt 1 ] ; then
cd ${cwd}
fi
fi

# Search for the file again, because grc might have renamed the file
FILE=$(find ${RUNDIR}/output -regex "${REGEX}")
FILE=$(find_one_file ${RUNDIR}/output -regex "${REGEX}")

# check presence of output file
if [ ! -s "${FILE}" ]; then
if [ "$VERBOSE" -gt 0 ]; then
echo "File $FILE does not exists or is zero size"
echo "File $FILE does not exist or is zero size"
fi
exit 20 # FAIL
fi
Expand Down
2 changes: 1 addition & 1 deletion checkers/existence_ncdf_out.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ if [ ! -d "${RUNDIR}" ] ; then
exit 20 # FAIL
fi

FILELIST=$(ls -1 ${RUNDIR}/output/l[bf]f*00.nc 2>/dev/null)
FILELIST=$(ls -1 ${RUNDIR}/output/l[bf]ff[0-9][0-9]*.nc 2>/dev/null)
if [ $? -ne 0 ] ; then
echo "No netCDF output file found" 1>&1
exit 20 # FAIL
Expand Down
4 changes: 3 additions & 1 deletion checkers/identical_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ def check():
if switch: # extract timestep
nlfile = switch # namelist file containing dt
try:
dt = float(get_param(rundir+nlfile, 'dt'))
dt = get_param(rundir+nlfile, 'dt')
assert(dt != '')
dt = float(dt)
except:
if verbose:
print header+'failed to extract dt from '+rundir+nlfile
Expand Down
10 changes: 8 additions & 2 deletions checkers/output_tolerance_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ def check():

# extract timestep
try:
dt = float(get_param(rundir+nlfile2, 'dt'))
dt = get_param(rundir+nlfile2, 'dt')
assert(dt != '')
dt = float(dt)
except:
if verbose:
print header+'failed to extract dt from '+rundir+nlfile2
Expand All @@ -66,13 +68,17 @@ def check():
# check for output lists
try:
nout_list = get_param(rundir+nlfile, lnout)
assert(nout_list != '')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks to the assert, I saw this is in fact not working in some tests, see
#37

nout_list = int(nout_list)
except:
if verbose:
print header+'no output lists in '+rundir+nlfile
return 20 # FAIL

# check if special testsuite output was activated in every output list
if get_param(rundir+nlfile, yuswitch, occurrence=nout_list) in ['.FALSE.', '.false.']:
yuswitch_value = get_param(rundir+nlfile, yuswitch, occurrence=nout_list)
assert(yuswitch_value != '')
if yuswitch_value in ['.FALSE.', '.false.']:
if verbose:
print yuswitch+' is set to .false. in '+rundir+nlfile+' for this simulation'
return 20 # FAIL
Expand Down
4 changes: 2 additions & 2 deletions checkers/run_success_check.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2

"""
COSMO TECHNICAL TESTSUITE
Expand Down Expand Up @@ -46,7 +46,7 @@ def run_checker():
OccurrenceCrashPattern( "Cleanup pattern", "(.*)^(.*)CLEAN(\s*)UP(.*)" )
]

cosmo_filechecker = FileChecker()
cosmo_filechecker = FileChecker(verbose)
cosmo_filechecker.add_pattern_list(patterns)
return cosmo_filechecker.check(logfile, verbose)

Expand Down
29 changes: 19 additions & 10 deletions checkers/tolerance_check.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2

"""
COSMO TECHNICAL TESTSUITE
Expand Down Expand Up @@ -46,7 +46,7 @@ def check():
icon = str_to_bool(env['ICON'])
yufile = env['YUFILE']

#check if namelist file with switch exists in namelistdir
# check if namelist file with switch exists in namelistdir
switch_path = namelistdir + switch
if not os.path.exists(switch_path):
if verbosity>0:
Expand All @@ -58,15 +58,16 @@ def check():
yufile2 = refoutdir + yufile
# check if special testsuite output was activated
if not icon:
if get_param(rundir+switch, yuswitch) in ['.FALSE.', '.false.']:
if verbosity:
print yuswitch +' is set to .false. in '+ rundir + switch +' for this simulation'
return 20 # FAIL

#check if tolerance file exists in namelistdir or type dir
yuswitch_value = get_param(rundir+switch, yuswitch)
assert(yuswitch_value != '')
if yuswitch_value in ['.FALSE.', '.false.']:
if verbosity:
print yuswitch +' is set to .false. in '+ rundir + switch +' for this simulation'
return 20 # FAIL

# check if tolerance file exists in namelistdir or type dir
# Note: the forcematch option gives precedence over the tolerance file
tolerance_path = namelistdir + tolerance

# The forcematch option gives precedence over the tolerance file
if forcematch:
thresh = """
minval = 0.0
Expand All @@ -79,6 +80,14 @@ def check():
print header + "unable to find tolerance file at " + tolerance
return 20 # FAIL

if (verbosity > 1):
print header + " yufile1: " + yufile1
print header + " yufile2: " + yufile2
if forcematch:
print header + " thresholds: " + "(force match)"
else:
print header + " thresholds: " + tolerance_path

try:
c = Compare(yufile1, yufile2, thresh)
if reset_thresholds:
Expand Down
3 changes: 3 additions & 0 deletions tests/regression/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
env
.coverage
htmlcov
3 changes: 3 additions & 0 deletions tests/regression/clean
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

\rm -rf work *.out *.pyc __pycache__ .pytest_cache data/ .coverage htmlcov
24 changes: 24 additions & 0 deletions tests/regression/config.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[ts_config]
# List of INPUT namelist
# Should be a python list format
l_files = ['INPUT_ORG','INPUT_ASS','INPUT_DIA','INPUT_DYN','INPUT_INI',\
'INPUT_IO','INPUT_PHY','INPUT_SAT','INPUT_POL','INPUT_IDEAL']
# File containing the parallelisation option (nproc ...)
par_file = INPUT_ORG
# File containing time step information
dt_file = INPUT_ORG
# File containing lasync_io flag
io_file = INPUT_IO
# File containing test result
res_file = TEST_RES
# File used for the tolerance checker
yufile = YUPRTEST
# List of dual parameters (which may be interchanged in the namelists)
# Should be a python list format
dual_params=[('nstop','hstop'),('hincrad','nincrad')]
# Where is the namelist switch for the testsuite output
nl_ts_switch = INPUT_DIA
# Where are all the namelists after execution
config_nl = YUSPECIF
# Is a perturbation necessary
pert_avail = True
1 change: 1 addition & 0 deletions tests/regression/config2.cfg
1 change: 1 addition & 0 deletions tests/regression/data_ref/basic/in_aux/aux_dummy
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This is a dummy aux file.
1 change: 1 addition & 0 deletions tests/regression/data_ref/basic/input/in_dummy
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This is a dummy input file.
4 changes: 4 additions & 0 deletions tests/regression/data_ref/basic/test_basic/INPUT_DIA
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
&diactl
ltestsuite=.TRUE.,
/END

6 changes: 6 additions & 0 deletions tests/regression/data_ref/basic/test_basic/INPUT_IO
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
&ioctl
ngribout=1,
/
&gribout
lcheck=.true.,
/
9 changes: 9 additions & 0 deletions tests/regression/data_ref/basic/test_basic/INPUT_ORG
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
&runctl
dt=60.0,
hstart=0.0,
hstop=2.0,
nprocx=1,
nprocy=1,
nprocio=0,
itype_pert=0,
/
7 changes: 7 additions & 0 deletions tests/regression/data_ref/basic/test_basic/TOLERANCE
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
minval = 1e-12
steps = 0 1 2 3 4 5 7 10 20 40 100
* = 0.00e+00 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 4.00e-11
eta = 0.00e+00 7.00e-15 7.00e-15 7.00e-15 7.00e-15 7.00e-15 7.00e-15 7.00e-15 9.00e-14 9.00e-14 2.00e-13
u = 0.00e+00 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 2.00e-13 4.00e-11
v = 0.00e+00 4.00e-14 4.00e-14 4.00e-14 4.00e-14 4.00e-14 4.00e-14 4.00e-14 4.00e-14 4.00e-14 4.00e-14
CHKDAT = 5.00e-05 1.00e-04 1.00e-01 1.00e+00 1.00e+00 1.00e+00 5.00e-05 1.00e-04 1.00e-01 1.00e+00 1.00e+00
1 change: 1 addition & 0 deletions tests/regression/data_ref/basic/test_basic/TOLERANCE2
10 changes: 10 additions & 0 deletions tests/regression/data_ref/basic/test_basic/YUCHKDAT
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@

Check the file data:
File: output/lfff00000120
ie_tot = 70 je_tot = 7 ke_tot = 80

var ee lev min imin jmin max imax jmax mean
u 13 1 -0.095288 0 0 0.003686 0 0 -0.014846
v 17 1 -0.113997 0 0 0.042260 0 0 -0.005102
eta 33 1 -0.169840 0 0 0.820299 0 0 0.055224
KE 102 1 0.000000 0 0 0.013075 0 0 0.001299
70 changes: 70 additions & 0 deletions tests/regression/data_ref/basic/test_basic/YUPRTEST
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# Experiment: Model
# ie_tot = 49 je_tot = 49 ke = 63
#
# var nt lev min imin jmin max imax jmax mean
u 0 1 0.000000000000000000E+00 0 0 0.000000000000000000E+00 0 0 0.000000000000000000E+00
v 0 1 0.000000000000000000E+00 0 0 0.000000000000000000E+00 0 0 0.000000000000000000E+00
eta 0 1 2.165689130516408471E-29 0 0 9.961278882600469453E-01 0 0 5.522382910304456799E-02
u 1 1 -3.849458898343552805E-03 0 0 3.857983612378607183E-03 0 0 -6.611470845315728807E-05
v 1 1 -3.835380936624159954E-03 0 0 3.849516122251034258E-03 0 0 -6.215758017089209848E-06
eta 1 1 2.345695763539030966E-29 0 0 9.953017139538279379E-01 0 0 5.522382910304456799E-02
u 2 1 -7.692658883466689350E-03 0 0 7.713223876914587773E-03 0 0 -1.324317283852256626E-04
v 2 1 -7.667989748219322492E-03 0 0 7.691689663357714063E-03 0 0 -1.225612305308798210E-05
eta 2 1 2.718304073672345290E-29 0 0 9.936507572269372979E-01 0 0 5.522382910304456799E-02
u 3 1 -1.152494372395403838E-02 0 0 1.156051196185309965E-02 0 0 -1.991318837089660722E-04
v 3 1 -1.149325898728429350E-02 0 0 1.152160487686155511E-02 0 0 -1.815950560166123610E-05
eta 3 1 3.309580697700215026E-29 0 0 9.911778070194778856E-01 0 0 5.522382910304458187E-02
u 4 1 -1.534167465409049837E-02 0 0 1.539399269279500583E-02 0 0 -2.663925937896652850E-04
v 4 1 -1.530663479479465149E-02 0 0 1.533436765403836666E-02 0 0 -2.396332646479783308E-05
eta 4 1 4.160699553335569057E-29 0 0 9.878870420813891107E-01 0 0 5.522382910304456799E-02
u 5 1 -1.913823830891700842E-02 0 0 1.920687789955326241E-02 0 0 -3.343861984290755728E-04
v 5 1 -1.910358449007975093E-02 0 0 1.912511457882534044E-02 0 0 -2.970382931367051131E-05
eta 5 1 5.330521103146963194E-29 0 0 9.837840200858807282E-01 0 0 5.522382910304456799E-02
u 6 1 -2.291005461310782407E-02 0 0 2.299120008110080646E-02 0 0 -4.032783810037476185E-04
v 6 1 -2.287960411729544896E-02 0 0 2.288902191552621632E-02 0 0 -3.541591222889588209E-05
eta 6 1 6.899267108538228284E-29 0 0 9.788756631372773631E-01 0 0 5.522382910304458187E-02
u 7 1 -2.665258460817618302E-02 0 0 2.673761756016777499E-02 0 0 -4.732267188548649963E-04
v 7 1 -2.663022593451126740E-02 0 0 2.662131450880340575E-02 0 0 -4.113298187172539634E-05
eta 7 1 8.973506632645738358E-29 0 0 9.731702397219794687E-01 0 0 5.522382910304456799E-02
u 8 1 -3.036133820253821350E-02 0 0 3.043528043192210084E-02 0 0 -5.443793868587664840E-04
v 8 1 -3.035102583148155397E-02 0 0 3.031727457561364872E-02 0 0 -4.688683339675258727E-05
eta 8 1 1.169274480257004141E-28 0 0 9.666773431636760394E-01 0 0 5.522382910304458187E-02
u 9 1 -3.403188182900170056E-02 0 0 3.407176440437785503E-02 0 0 -6.168740363169907589E-04
v 9 1 -3.403763066223040173E-02 0 0 3.397225036893154748E-02 0 0 -5.270755859825849830E-05
eta 9 1 1.523799762389538719E-28 0 0 9.594078666563252700E-01 0 0 5.522382910304456799E-02
u 10 1 -3.765984599430126367E-02 0 0 3.763307716062441743E-02 0 0 -6.908368669996921025E-04
v 10 1 -3.768572547864499495E-02 0 0 3.758166469333560494E-02 0 0 -5.862348410336022702E-05
eta 10 1 1.984284891171441316E-28 0 0 9.513739749608354224E-01 0 0 5.522382910304456799E-02
u 20 1 -7.069654193677141207E-02 0 0 6.660346746990539191E-02 0 0 -1.529986316873608405E-03
v 20 1 -7.115477628695109080E-02 0 0 7.034133622904563399E-02 0 0 -1.280182359828690487E-04
eta 20 1 2.578246346016506010E-27 0 0 8.325055673301089421E-01 0 0 5.522382910304455411E-02
u 30 1 -9.541409038619252936E-02 0 0 8.309606731653416956E-02 0 0 -2.567306370223154319E-03
v 30 1 -9.671195395352308322E-02 0 0 9.475093398228344455E-02 0 0 -2.272689720382698960E-04
eta 30 1 2.987407885692794554E-26 0 0 6.584545930077215781E-01 0 0 5.522382910304455411E-02
u 40 1 -1.100508530814234553E-01 0 0 7.719553232820777766E-02 0 0 -3.782634643842510552E-03
v 40 1 -1.119440132384657838E-01 0 0 1.106122350742410054E-01 0 0 -3.707919296440171297E-04
eta 40 1 3.181875335092288452E-25 0 0 6.659314870486774396E-01 0 0 5.522382910304454717E-02
u 50 1 -1.160769113112332385E-01 0 0 5.913819249444620535E-02 0 0 -5.148142292701780864E-03
v 50 1 -1.177561899680515944E-01 0 0 1.158975975538960868E-01 0 0 -5.782977354372791871E-04
eta 50 1 3.153598733213607455E-24 0 0 6.499141669012825639E-01 0 0 5.522382910304454717E-02
u 60 1 -1.133924228931027650E-01 0 0 3.604013761737683008E-02 0 0 -6.628222144932366909E-03
v 60 1 -1.162515148477110405E-01 0 0 1.125574336607608333E-01 0 0 -8.724761947243736288E-04
eta 60 1 2.925378209752125347E-23 0 0 5.993805666152879352E-01 0 0 5.522382910304454717E-02
u 70 1 -1.069155152024322680E-01 0 0 1.916776659423633650E-02 0 0 -8.166774150834121143E-03
v 70 1 -1.216608228562506000E-01 0 0 1.218252845732289841E-01 0 0 -1.273797721918859017E-03
eta 70 1 -6.665379545771726866E-02 0 0 5.499291077862874522E-01 0 0 5.522382910304451942E-02
u 80 1 -9.938236557169073671E-02 0 0 1.014516363878045432E-02 0 0 -9.703460325732330555E-03
v 80 1 -1.258966613369230658E-01 0 0 1.219229518559721004E-01 0 0 -1.798508543845048300E-03
eta 80 1 -1.465114359457190263E-01 0 0 5.556185440210257287E-01 0 0 5.522382910304451248E-02
u 90 1 -9.275845828397710768E-02 0 0 1.496996502908658298E-03 0 0 -1.118338017485132931E-02
v 90 1 -1.261216544495663561E-01 0 0 1.130126090780570053E-01 0 0 -2.454872303062005916E-03
eta 90 1 -1.765983184052934141E-01 0 0 6.322587299764972357E-01 0 0 5.522382910304451942E-02
u 100 1 -9.012746608338956245E-02 0 0 5.852564112425879790E-05 0 0 -1.255587669827768794E-02
v 100 1 -1.234004314201772035E-01 0 0 9.676074828756117685E-02 0 0 -3.238566829864447574E-03
eta 100 1 -1.778268219325232491E-01 0 0 7.293143756723945437E-01 0 0 5.522382910304453330E-02
u 110 1 -9.546339188624054783E-02 0 0 1.211407134026049710E-04 0 0 -1.378330988624681432E-02
v 110 1 -1.189629942758403808E-01 0 0 7.049313933752406058E-02 0 0 -4.131445647557008966E-03
eta 110 1 -1.608210421961591419E-01 0 0 7.962781502016331725E-01 0 0 5.522382910304451942E-02
u 120 1 -9.528805440797019632E-02 0 0 3.686192315303329480E-03 0 0 -1.484643587868229343E-02
v 120 1 -1.139968580611173399E-01 0 0 4.225988463019059865E-02 0 0 -5.102295963162439309E-03
eta 120 1 -1.698400505108406400E-01 0 0 8.202991184514930589E-01 0 0 5.522382910304451248E-02
12 changes: 12 additions & 0 deletions tests/regression/data_ref/basic/test_basic/YUSPECIF
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@

0 The NAMELIST variables were specified as follows:
=================================================


0 NAMELIST: runctl
-----------------

Variable Actual Value Default Value Format
nprocx 4 0 I
nprocy 4 0 I
nprocio 0 0 I
Loading