diff --git a/docs/source/config.rst b/docs/source/config.rst index ef7b9451..2f279f0b 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -144,7 +144,7 @@ configuration. ``input`` Listing of the directories containing model input fields, linked to the experiment during setup. This can either be the name of a directory in the - laboratory's ``input`` directory:: + control directory or in laboratory's ``input`` directory:: input: core_inputs diff --git a/payu/models/model.py b/payu/models/model.py index 1b9da485..729d7681 100644 --- a/payu/models/model.py +++ b/payu/models/model.py @@ -3,7 +3,7 @@ :copyright: Copyright 2011 Marshall Ward, see AUTHORS for details :license: Apache License, Version 2.0, see LICENSE for details """ -import errno + import os import shutil import shlex @@ -129,14 +129,20 @@ def set_input_paths(self): self.input_paths = [] for input_dir in input_dirs: - # First test for absolute path + # First test if path exists if os.path.exists(input_dir): + # Resolve symlinks and relative paths + input_dir = os.path.realpath(input_dir) + self.input_paths.append(input_dir) else: # Test for path relative to /${lab_path}/input/${model_name} assert self.input_basepath rel_path = os.path.join(self.input_basepath, input_dir) if os.path.exists(rel_path): + # Resolve symlinks + rel_path = os.path.realpath(rel_path) + self.input_paths.append(rel_path) else: sys.exit('payu: error: Input directory {0} not found; ' @@ -231,17 +237,15 @@ def setup_configuration_files(self): path to work path""" for f_name in self.config_files: f_path = os.path.join(self.control_path, f_name) - shutil.copy(f_path, self.work_path) + shutil.copy(f_path, self.work_path, follow_symlinks=True) for f_name in self.optional_config_files: f_path = os.path.join(self.control_path, f_name) try: - shutil.copy(f_path, self.work_path) - except IOError as exc: - if exc.errno == errno.ENOENT: - pass - else: - raise + shutil.copy(f_path, self.work_path, follow_symlinks=True) + except FileNotFoundError: + # Ignore missing files for optional config files + pass def setup(self): diff --git a/payu/models/test.py b/payu/models/test.py index a6c85511..ec7aff19 100644 --- a/payu/models/test.py +++ b/payu/models/test.py @@ -15,6 +15,7 @@ 'diag', 'input.nml' ] +optional_config_files = ['opt_data'] class Test(Model): @@ -29,3 +30,4 @@ def __init__(self, expt, name, config): self.default_exec = 'test.exe' self.config_files = config_files + self.optional_config_files = optional_config_files diff --git a/test/test_pbs.py b/test/test_pbs.py index 1ee67b12..7389c17f 100644 --- a/test/test_pbs.py +++ b/test/test_pbs.py @@ -1,5 +1,6 @@ import argparse from argparse import Namespace +import copy import os from pathlib import Path import shutil @@ -18,13 +19,16 @@ from .common import cd, make_random_file, get_manifests from .common import tmpdir, ctrldir, labdir, workdir, payudir -from .common import config, sweep_work, payu_init, payu_setup +from .common import sweep_work, payu_init, payu_setup +from .common import config as original_config from .common import write_config from .common import make_exe, make_inputs, make_restarts from .common import make_payu_exe, make_all_files verbose = True +config = copy.deepcopy(original_config) + def setup_module(module): """ diff --git a/test/test_setup.py b/test/test_setup.py index 70465193..857ada83 100644 --- a/test/test_setup.py +++ b/test/test_setup.py @@ -1,84 +1,45 @@ import copy -import os from pathlib import Path -import pdb import pytest import shutil from unittest.mock import patch -import yaml import payu -import payu.models.test from .common import cd, make_random_file, get_manifests from .common import tmpdir, ctrldir, labdir, workdir -from .common import sweep_work, payu_init, payu_setup +from .common import payu_init, payu_setup from .common import config as config_orig from .common import write_config -from .common import make_exe, make_inputs, make_restarts, make_all_files +from .common import make_exe, make_inputs -verbose = True +# Config files in the test model driver +CONFIG_FILES = ['data', 'diag', 'input.nml'] +OPTIONAL_CONFIG_FILES = ['opt_data'] +INPUT_NML_FILENAME = 'input.nml' -config = copy.deepcopy(config_orig) - - -def make_config_files(): - """ - Create files required for test model - """ - - config_files = payu.models.test.config_files - for file in config_files: - make_random_file(ctrldir/file, 29) - - -def setup_module(module): - """ - Put any test-wide setup code in here, e.g. creating test files - """ - if verbose: - print("setup_module module:%s" % module.__name__) - - # Should be taken care of by teardown, in case remnants lying around - try: - shutil.rmtree(tmpdir) - except FileNotFoundError: - pass +@pytest.fixture(autouse=True) +def setup_and_teardown(): + # Create tmp, lab and control directories try: tmpdir.mkdir() labdir.mkdir() ctrldir.mkdir() - make_all_files() except Exception as e: print(e) - write_config(config) - - -def teardown_module(module): - """ - Put any test-wide teardown code in here, e.g. removing test outputs - """ - if verbose: - print("teardown_module module:%s" % module.__name__) + yield + # Remove tmp directory try: - # shutil.rmtree(tmpdir) - print('removing tmp') + shutil.rmtree(tmpdir) except Exception as e: print(e) -# These are integration tests. They have an undesirable dependence on each -# other. It would be possible to make them independent, but then they'd -# be reproducing previous "tests", like init. So this design is deliberate -# but compromised. It means when running an error in one test can cascade -# and cause other tests to fail. -# -# Unfortunate but there you go. - def test_init(): + write_config(config_orig) # Initialise a payu laboratory with cd(ctrldir): @@ -89,54 +50,131 @@ def test_init(): assert((labdir / subdir).is_dir()) -def test_setup(): +def make_config_files(): + """ + Create files required for test model + """ + for file in CONFIG_FILES: + make_random_file(ctrldir/file, 29) - # Create some input and executable files - make_inputs() - make_exe() - bindir = labdir / 'bin' - exe = config['exe'] +def run_payu_setup(config=config_orig, create_inputs=False, + create_config_files=True): + """Helper function to write config.yaml files, make inputs, + config files and run experiment setup""" + # Setup files + write_config(config) + make_exe() + if create_inputs: + make_inputs() + if create_config_files: + make_config_files() - make_config_files() + # Initialise a payu laboratory + with cd(ctrldir): + payu_init(None, None, str(labdir)) - # Run setup + # Run payu setup payu_setup(lab_path=str(labdir)) - assert(workdir.is_symlink()) - assert(workdir.is_dir()) - assert((workdir/exe).resolve() == (bindir/exe).resolve()) - workdirfull = workdir.resolve() - config_files = payu.models.test.config_files +def test_setup_configuration_files(): + """Test model config_files are copied to work directory, + and that any symlinks are followed""" + # Create configuration files + all_config_files = CONFIG_FILES + OPTIONAL_CONFIG_FILES + for file in all_config_files: + if file != INPUT_NML_FILENAME: + make_random_file(ctrldir / file, 8) - for f in config_files + ['config.yaml']: - assert((workdir/f).is_file()) + # For input.nml, create a file symlink in control directory + input_nml_realpath = tmpdir / INPUT_NML_FILENAME + input_nml_symlink = ctrldir / INPUT_NML_FILENAME + make_random_file(input_nml_realpath, 8) + input_nml_symlink.symlink_to(input_nml_realpath) + assert input_nml_symlink.is_symlink() - for i in range(1, 4): - assert((workdir/'input_00{i}.bin'.format(i=i)).stat().st_size - == 1000**2 + i) + # Run payu setup + run_payu_setup(create_inputs=True) + + # Check config files have been copied to work path + for file in all_config_files + ['config.yaml']: + filepath = workdir / file + assert filepath.exists() and filepath.is_file() + assert not filepath.is_symlink() + assert filepath.read_bytes() == (ctrldir / file).read_bytes() - with pytest.raises(SystemExit, - match="work path already exists") as setup_error: - payu_setup(lab_path=str(labdir), sweep=False, force=False) - assert setup_error.type == SystemExit - payu_setup(lab_path=str(labdir), sweep=False, force=True) +@pytest.mark.parametrize( + "input_path, is_symlink, is_absolute", + [ + (labdir / 'input' / 'lab_inputs', False, False), + (ctrldir / 'ctrl_inputs', False, False), + (tmpdir / 'symlinked_inputs', True, False), + (tmpdir / 'tmp_inputs', False, True) + ] +) +def test_setup_inputs(input_path, is_symlink, is_absolute): + """Test inputs are symlinked to work directory, + and added in input manifest""" + # Make inputs + input_path.mkdir(parents=True, exist_ok=True) + for i in range(1, 4): + make_random_file(input_path / f'input_00{i}.bin', 1000**2 + i) - assert(workdir.is_symlink()) - assert(workdir.is_dir()) - assert((workdir/exe).resolve() == (bindir/exe).resolve()) - workdirfull = workdir.resolve() + if is_symlink: + # Create an input symlink in control directory + (ctrldir / input_path.name).symlink_to(input_path) - config_files = payu.models.test.config_files + # Modify config to specify input path + config = copy.deepcopy(config_orig) + config['input'] = str(input_path) if is_absolute else input_path.name - for f in config_files + ['config.yaml']: - assert((workdir/f).is_file()) + # Run payu setup + run_payu_setup(config=config, create_config_files=True) + input_manifest = get_manifests(ctrldir/'manifests')['input.yaml'] for i in range(1, 4): - assert((workdir/'input_00{i}.bin'.format(i=i)).stat().st_size - == 1000**2 + i) + filename = f'input_00{i}.bin' + work_input = workdir / filename + + # Check file exists and file size is expected + assert work_input.exists() and work_input.is_symlink() + assert work_input.stat().st_size == 1000**2 + i + + # Check relative input path is added to manifest + filepath = str(Path('work') / filename) + assert filepath in input_manifest + + # Check manifest fullpath + manifest_fullpath = input_manifest[filepath]['fullpath'] + assert manifest_fullpath == str(input_path / filename) + + # Check fullpath is a resolved path + assert Path(manifest_fullpath).is_absolute() + assert not Path(manifest_fullpath).is_symlink() + + +def test_setup(): + """Test work directory and executable are setup as expected, + and re-running setup requires a force=True""" + run_payu_setup(create_inputs=True, create_config_files=True) + + assert workdir.is_symlink and workdir.is_dir() + + # Check executable symlink is in work directory + bin_exe = labdir / 'bin' / config_orig['exe'] + work_exe = workdir / config_orig['exe'] + assert work_exe.exists() and work_exe.is_symlink() + assert work_exe.resolve() == bin_exe.resolve() + + # Re-run setup - expect an error + with pytest.raises(SystemExit, + match="work path already exists") as setup_error: + payu_setup(lab_path=str(labdir), sweep=False, force=False) + assert setup_error.type == SystemExit + + assert workdir.is_symlink and workdir.is_dir() @pytest.mark.parametrize(