Support symbolic links in input files (#518)

* Support symbolic links in model inputs
* Extend payu setup tests for symlinked config and inputs files

docs/source/config.rst

@@ -144,7 +144,7 @@ configuration.
 ``input``
    Listing of the directories containing model input fields, linked to the
    experiment during setup. This can either be the name of a directory in the
-   laboratory's ``input`` directory::
+   control directory or in laboratory's ``input`` directory::
 
       input: core_inputs
 

payu/models/model.py

@@ -3,7 +3,7 @@
 :copyright: Copyright 2011 Marshall Ward, see AUTHORS for details
 :license: Apache License, Version 2.0, see LICENSE for details
 """
-import errno
+
 import os
 import shutil
 import shlex
@@ -129,14 +129,20 @@ def set_input_paths(self):
         self.input_paths = []
         for input_dir in input_dirs:
 
-            # First test for absolute path
+            # First test if path exists
             if os.path.exists(input_dir):
+                # Resolve symlinks and relative paths
+                input_dir = os.path.realpath(input_dir)
+
                 self.input_paths.append(input_dir)
             else:
                 # Test for path relative to /${lab_path}/input/${model_name}
                 assert self.input_basepath
                 rel_path = os.path.join(self.input_basepath, input_dir)
                 if os.path.exists(rel_path):
+                    # Resolve symlinks
+                    rel_path = os.path.realpath(rel_path)
+
                     self.input_paths.append(rel_path)
                 else:
                     sys.exit('payu: error: Input directory {0} not found; '
@@ -231,17 +237,15 @@ def setup_configuration_files(self):
          path to work path"""
         for f_name in self.config_files:
             f_path = os.path.join(self.control_path, f_name)
-            shutil.copy(f_path, self.work_path)
+            shutil.copy(f_path, self.work_path, follow_symlinks=True)
 
         for f_name in self.optional_config_files:
             f_path = os.path.join(self.control_path, f_name)
             try:
-                shutil.copy(f_path, self.work_path)
-            except IOError as exc:
-                if exc.errno == errno.ENOENT:
-                    pass
-                else:
-                    raise
+                shutil.copy(f_path, self.work_path, follow_symlinks=True)
+            except FileNotFoundError:
+                # Ignore missing files for optional config files
+                pass
 
     def setup(self):
 

payu/models/test.py

@@ -15,6 +15,7 @@
             'diag',
             'input.nml'
         ]
+optional_config_files = ['opt_data']
 
 
 class Test(Model):
@@ -29,3 +30,4 @@ def __init__(self, expt, name, config):
         self.default_exec = 'test.exe'
 
         self.config_files = config_files
+        self.optional_config_files = optional_config_files

test/test_pbs.py

@@ -1,5 +1,6 @@
 import argparse
 from argparse import Namespace
+import copy
 import os
 from pathlib import Path
 import shutil
@@ -18,13 +19,16 @@
 
 from .common import cd, make_random_file, get_manifests
 from .common import tmpdir, ctrldir, labdir, workdir, payudir
-from .common import config, sweep_work, payu_init, payu_setup
+from .common import sweep_work, payu_init, payu_setup
+from .common import config as original_config
 from .common import write_config
 from .common import make_exe, make_inputs, make_restarts
 from .common import make_payu_exe, make_all_files
 
 verbose = True
 
+config = copy.deepcopy(original_config)
+
 
 def setup_module(module):
     """

test/test_setup.py

@@ -1,84 +1,45 @@
 import copy
-import os
 from pathlib import Path
-import pdb
 import pytest
 import shutil
 from unittest.mock import patch
-import yaml
 
 import payu
-import payu.models.test
 
 from .common import cd, make_random_file, get_manifests
 from .common import tmpdir, ctrldir, labdir, workdir
-from .common import sweep_work, payu_init, payu_setup
+from .common import payu_init, payu_setup
 from .common import config as config_orig
 from .common import write_config
-from .common import make_exe, make_inputs, make_restarts, make_all_files
+from .common import make_exe, make_inputs
 
-verbose = True
+# Config files in the test model driver
+CONFIG_FILES = ['data', 'diag', 'input.nml']
+OPTIONAL_CONFIG_FILES = ['opt_data']
+INPUT_NML_FILENAME = 'input.nml'
 
-config = copy.deepcopy(config_orig)
-
-
-def make_config_files():
-    """
-    Create files required for test model
-    """
-
-    config_files = payu.models.test.config_files
-    for file in config_files:
-        make_random_file(ctrldir/file, 29)
-
-
-def setup_module(module):
-    """
-    Put any test-wide setup code in here, e.g. creating test files
-    """
-    if verbose:
-        print("setup_module      module:%s" % module.__name__)
-
-    # Should be taken care of by teardown, in case remnants lying around
-    try:
-        shutil.rmtree(tmpdir)
-    except FileNotFoundError:
-        pass
 
+@pytest.fixture(autouse=True)
+def setup_and_teardown():
+    # Create tmp, lab and control directories
     try:
         tmpdir.mkdir()
         labdir.mkdir()
         ctrldir.mkdir()
-        make_all_files()
     except Exception as e:
         print(e)
 
-    write_config(config)
-
-
-def teardown_module(module):
-    """
-    Put any test-wide teardown code in here, e.g. removing test outputs
-    """
-    if verbose:
-        print("teardown_module   module:%s" % module.__name__)
+    yield
 
+    # Remove tmp directory
     try:
-        # shutil.rmtree(tmpdir)
-        print('removing tmp')
+        shutil.rmtree(tmpdir)
     except Exception as e:
         print(e)
 
-# These are integration tests. They have an undesirable dependence on each
-# other. It would be possible to make them independent, but then they'd
-# be reproducing previous "tests", like init. So this design is deliberate
-# but compromised. It means when running an error in one test can cascade
-# and cause other tests to fail.
-#
-# Unfortunate but there you go.
-
 
 def test_init():
+    write_config(config_orig)
 
     # Initialise a payu laboratory
     with cd(ctrldir):
@@ -89,54 +50,131 @@ def test_init():
         assert((labdir / subdir).is_dir())
 
 
-def test_setup():
+def make_config_files():
+    """
+    Create files required for test model
+    """
+    for file in CONFIG_FILES:
+        make_random_file(ctrldir/file, 29)
 
-    # Create some input and executable files
-    make_inputs()
-    make_exe()
 
-    bindir = labdir / 'bin'
-    exe = config['exe']
+def run_payu_setup(config=config_orig, create_inputs=False,
+                   create_config_files=True):
+    """Helper function to write config.yaml files, make inputs,
+    config files and run experiment setup"""
+    # Setup files
+    write_config(config)
+    make_exe()
+    if create_inputs:
+        make_inputs()
+    if create_config_files:
+        make_config_files()
 
-    make_config_files()
+    # Initialise a payu laboratory
+    with cd(ctrldir):
+        payu_init(None, None, str(labdir))
 
-    # Run setup
+    # Run payu setup
     payu_setup(lab_path=str(labdir))
 
-    assert(workdir.is_symlink())
-    assert(workdir.is_dir())
-    assert((workdir/exe).resolve() == (bindir/exe).resolve())
-    workdirfull = workdir.resolve()
 
-    config_files = payu.models.test.config_files
+def test_setup_configuration_files():
+    """Test model config_files are copied to work directory,
+    and that any symlinks are followed"""
+    # Create configuration files
+    all_config_files = CONFIG_FILES + OPTIONAL_CONFIG_FILES
+    for file in all_config_files:
+        if file != INPUT_NML_FILENAME:
+            make_random_file(ctrldir / file, 8)
 
-    for f in config_files + ['config.yaml']:
-        assert((workdir/f).is_file())
+    # For input.nml, create a file symlink in control directory
+    input_nml_realpath = tmpdir / INPUT_NML_FILENAME
+    input_nml_symlink = ctrldir / INPUT_NML_FILENAME
+    make_random_file(input_nml_realpath, 8)
+    input_nml_symlink.symlink_to(input_nml_realpath)
+    assert input_nml_symlink.is_symlink()
 
-    for i in range(1, 4):
-        assert((workdir/'input_00{i}.bin'.format(i=i)).stat().st_size
-               == 1000**2 + i)
+    # Run payu setup
+    run_payu_setup(create_inputs=True)
+
+    # Check config files have been copied to work path
+    for file in all_config_files + ['config.yaml']:
+        filepath = workdir / file
+        assert filepath.exists() and filepath.is_file()
+        assert not filepath.is_symlink()
+        assert filepath.read_bytes() == (ctrldir / file).read_bytes()
 
-    with pytest.raises(SystemExit,
-                       match="work path already exists") as setup_error:
-        payu_setup(lab_path=str(labdir), sweep=False, force=False)
-    assert setup_error.type == SystemExit
 
-    payu_setup(lab_path=str(labdir), sweep=False, force=True)
+@pytest.mark.parametrize(
+    "input_path, is_symlink, is_absolute",
+    [
+        (labdir / 'input' / 'lab_inputs', False, False),
+        (ctrldir / 'ctrl_inputs', False, False),
+        (tmpdir / 'symlinked_inputs', True, False),
+        (tmpdir / 'tmp_inputs', False, True)
+    ]
+)
+def test_setup_inputs(input_path, is_symlink, is_absolute):
+    """Test inputs are symlinked to work directory,
+    and added in input manifest"""
+    # Make inputs
+    input_path.mkdir(parents=True, exist_ok=True)
+    for i in range(1, 4):
+        make_random_file(input_path / f'input_00{i}.bin', 1000**2 + i)
 
-    assert(workdir.is_symlink())
-    assert(workdir.is_dir())
-    assert((workdir/exe).resolve() == (bindir/exe).resolve())
-    workdirfull = workdir.resolve()
+    if is_symlink:
+        # Create an input symlink in control directory
+        (ctrldir / input_path.name).symlink_to(input_path)
 
-    config_files = payu.models.test.config_files
+    # Modify config to specify input path
+    config = copy.deepcopy(config_orig)
+    config['input'] = str(input_path) if is_absolute else input_path.name
 
-    for f in config_files + ['config.yaml']:
-        assert((workdir/f).is_file())
+    # Run payu setup
+    run_payu_setup(config=config, create_config_files=True)
 
+    input_manifest = get_manifests(ctrldir/'manifests')['input.yaml']
     for i in range(1, 4):
-        assert((workdir/'input_00{i}.bin'.format(i=i)).stat().st_size
-               == 1000**2 + i)
+        filename = f'input_00{i}.bin'
+        work_input = workdir / filename
+
+        # Check file exists and file size is expected
+        assert work_input.exists() and work_input.is_symlink()
+        assert work_input.stat().st_size == 1000**2 + i
+
+        # Check relative input path is added to manifest
+        filepath = str(Path('work') / filename)
+        assert filepath in input_manifest
+
+        # Check manifest fullpath
+        manifest_fullpath = input_manifest[filepath]['fullpath']
+        assert manifest_fullpath == str(input_path / filename)
+
+        # Check fullpath is a resolved path
+        assert Path(manifest_fullpath).is_absolute()
+        assert not Path(manifest_fullpath).is_symlink()
+
+
+def test_setup():
+    """Test work directory and executable are setup as expected,
+    and re-running setup requires a force=True"""
+    run_payu_setup(create_inputs=True, create_config_files=True)
+
+    assert workdir.is_symlink and workdir.is_dir()
+
+    # Check executable symlink is in work directory
+    bin_exe = labdir / 'bin' / config_orig['exe']
+    work_exe = workdir / config_orig['exe']
+    assert work_exe.exists() and work_exe.is_symlink()
+    assert work_exe.resolve() == bin_exe.resolve()
+
+    # Re-run setup - expect an error
+    with pytest.raises(SystemExit,
+                       match="work path already exists") as setup_error:
+        payu_setup(lab_path=str(labdir), sweep=False, force=False)
+    assert setup_error.type == SystemExit
+
+    assert workdir.is_symlink and workdir.is_dir()
 
 
 @pytest.mark.parametrize(