explicitely only check for file presence instead of checking contents…

… + improving error messages in tests (#37)

{{cookiecutter.project_slug}}/ci/gitlab-ci.yml

@@ -22,6 +22,7 @@ before_script:
     - DOCKER_modern
   script:
     - echo "$SC_CI_TESTING_REGISTRY_PASSWORD" | docker login -u "$SC_CI_TESTING_REGISTRY_USER" --password-stdin $SC_CI_TESTING_REGISTRY
+    - echo "$SC_CI_MASTER_REGISTRY_PASSWORD" | docker login -u "$SC_CI_MASTER_REGISTRY_USER" --password-stdin $SC_CI_MASTER_REGISTRY
     - cd $SC_CI_{{ cookiecutter.project_package_name.upper() }}_LOCATION
     - export DOCKER_REGISTRY=$SC_CI_MASTER_REGISTRY
     - make pull-latest || true
@@ -87,6 +88,7 @@ remove_{{ cookiecutter.project_slug }}_builds:
   needs: ["{{ cookiecutter.project_slug }}-build"]
   script:
     - echo "$SC_CI_TESTING_REGISTRY_PASSWORD" | docker login -u "$SC_CI_TESTING_REGISTRY_USER" --password-stdin $SC_CI_TESTING_REGISTRY
+    - echo "$SC_CI_MASTER_REGISTRY_PASSWORD" | docker login -u "$SC_CI_MASTER_REGISTRY_USER" --password-stdin $SC_CI_MASTER_REGISTRY
     - cd $SC_CI_{{ cookiecutter.project_package_name.upper() }}_LOCATION
     - export DOCKER_REGISTRY=$SC_CI_TEST_IMAGE_PREFIX
     - make pull-latest

{{cookiecutter.project_slug}}/tests/integration/test_docker_container.py

@@ -56,6 +56,7 @@ def docker_container(validation_folders: Dict, host_folders: Dict, docker_client
     shutil.copytree(validation_folders["input"], host_folders["input"])
     assert Path(host_folders["input"]).exists()
     # run the container (this may take some time)
+    container = None
     try:
         volumes = {host_folders[folder]: {"bind": container_variables["{}_FOLDER".format(
             str(folder).upper())]} for folder in _FOLDER_NAMES}
@@ -75,10 +76,11 @@ def docker_container(validation_folders: Dict, host_folders: Dict, docker_client
         pytest.fail("The container stopped with exit code {}\n\n\ncommand:\n {}, \n\n\nlog:\n{}".format(exc.exit_status,
                                                                                                         exc.command, pformat(
                                                                                                             (container.logs(timestamps=True).decode("UTF-8")).split("\n"), width=200
-                                                                                                        )))
+                                                                                                        ) if container else ""))
     finally:
         # cleanup
-        container.remove()
+        if container:
+            container.remove()
 
 
 def _convert_to_simcore_labels(image_labels: Dict) -> Dict:
@@ -102,23 +104,25 @@ def test_run_container(validation_folders: Dict, host_folders: Dict, docker_cont
             x.name for x in validation_folders[folder].iterdir() if not ".gitkeep" in x.name]
         for file_name in list_of_files:
             assert Path(host_folders[folder] / file_name).exists(
-            ), "missing files in {}".format(host_folders[folder])
-        match, mismatch, errors = filecmp.cmpfiles(
+            ), f"{file_name} is missing from {host_folders[folder]}"
+
+        # we look for missing files only. contents is the responsibility of the service creator
+        _, _, errors = filecmp.cmpfiles(
+            host_folders[folder], validation_folders[folder], list_of_files, shallow=True)
+        assert not errors, f"{errors} are missing in {host_folders[folder]}"
+
+        if folder == "input":
+            continue
+        # test if the generated files are the ones expected
+        list_of_files = [
+            x.name for x in host_folders[folder].iterdir() if not ".gitkeep" in x.name]
+        for file_name in list_of_files:
+            assert Path(validation_folders[folder] / file_name).exists(
+            ), "{} is not present in {}".format(file_name, validation_folders[folder])
+        _, _, errors = filecmp.cmpfiles(
             host_folders[folder], validation_folders[folder], list_of_files, shallow=False)
-        # assert not mismatch, "wrong/incorrect files in {}".format(host_folders[folder])
-        assert not errors, "missing files in {}".format(host_folders[folder])
-        # test if the files that are there are matching the ones that should be
-        if folder != "input":
-            list_of_files = [
-                x.name for x in host_folders[folder].iterdir() if not ".gitkeep" in x.name]
-            for file_name in list_of_files:
-                assert Path(validation_folders[folder] / file_name).exists(
-                ), "{} is not present in {}".format(file_name, validation_folders[folder])
-            match, mismatch, errors = filecmp.cmpfiles(
-                host_folders[folder], validation_folders[folder], list_of_files, shallow=False)
-            # assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder])
-            assert not errors, "too many files in {}".format(
-                host_folders[folder])
+        # assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder])
+        assert not errors, f"{errors} should not be available in {host_folders[folder]}"
 
     # check the output is correct based on container labels
     output_cfg = {}

{{cookiecutter.project_slug}}/tests/unit/test_validation_data.py

@@ -3,7 +3,7 @@
 # pylint:disable=redefined-outer-name
 import json
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Iterator, Optional
 
 import pytest
 
@@ -30,7 +30,7 @@ def validation_folder(validation_dir: Path, port_type: str) -> Path:
 
 
 @pytest.fixture
-def validation_cfg(validation_dir: Path, port_type: str) -> Dict:
+def validation_cfg(validation_dir: Path, port_type: str) -> Optional[Dict]:
     validation_file = validation_dir / \
         port_type / (f"{port_type}s.json")
     if validation_file.exists():
@@ -40,7 +40,7 @@ def validation_cfg(validation_dir: Path, port_type: str) -> Dict:
     return None
 
 
-def _find_key_in_cfg(filename: str, value: Dict) -> str:
+def _find_key_in_cfg(filename: str, value: Dict) -> Iterator[str]:
     for k, v in value.items():
         if k == filename:
             if isinstance(v, dict):
@@ -64,19 +64,21 @@ def test_validation_data_follows_definition(label_cfg: Dict, validation_cfg: Dic
         # rationale: files are on their own and other types are in inputs.json
         if not "data:" in value["type"]:
             # check that keys are available
-            assert key in validation_cfg
+            assert key in validation_cfg, f"missing {key} in validation config file"
         else:
             # it's a file and it should be in the folder as well using key as the filename
             filename_to_look_for = key
             if "fileToKeyMap" in value:
                 # ...or there is a mapping
                 assert len(value["fileToKeyMap"]) > 0
                 for filename, mapped_value in value["fileToKeyMap"].items():
-                    assert mapped_value == key
+                    assert mapped_value == key, f"file to key map for {key} has an incorrectly set {mapped_value}, it should be equal to {key}"
                     filename_to_look_for = filename
-                    assert (validation_folder / filename_to_look_for).exists()
+                    assert (validation_folder / filename_to_look_for).exists(
+                    ), f"{filename_to_look_for} is missing from {validation_folder}"
             else:
-                assert (validation_folder / filename_to_look_for).exists()
+                assert (validation_folder / filename_to_look_for).exists(
+                ), f"{filename_to_look_for} is missing from {validation_folder}"
 
     if validation_cfg:
         for key, value in validation_cfg.items():
@@ -90,14 +92,15 @@ def test_validation_data_follows_definition(label_cfg: Dict, validation_cfg: Dic
             }
             if not "data:" in label_cfg[key]["type"]:
                 # check the type is correct
-                assert isinstance(value, label2types[label_cfg[key]["type"]])
+                assert isinstance(value, label2types[label_cfg[key]["type"]]
+                                  ), f"{value} has not the expected type {label2types[label_cfg[key]['type']]}"
 
     for path in validation_folder.glob("**/*"):
         if path.name in ["inputs.json", "outputs.json", ".gitkeep"]:
             continue
-        assert path.is_file()
+        assert path.is_file(), f"{path} is not a file!"
         filename = path.name
         # this filename shall be available as a key in the labels somewhere
         key = next(_find_key_in_cfg(str(filename), label_cfg))
 
-        assert key in label_cfg
+        assert key in label_cfg, f"{key} was not found in {label_cfg}"