test: include test data from esm-tools

tests/configs/test_config_fesom_2p6_pimesh.yaml

@@ -0,0 +1,23 @@
+pymorize:
+  warn_on_no_rule: False
+  parallel: False
+general:
+  name: "fesom_2p6_pimesh"
+  description: "This is a test configuration using esm-tools generated test data on PI Mesh"
+  maintainer: "pgierz"
+  email: "[email protected]"
+  cmor_version: "CMIP6"
+  mip: "CMIP"
+  frequency: "mon"
+  CMIP_Tables_Dir: "./cmip6-cmor-tables/Tables"
+rules:
+  - name: "temp"
+    experiment_id: "piControl"
+    output_directory: "./output"
+    source_id: "FESOM"
+    variant_label: "r1i1p1f1"
+    inputs:
+      - path: "REPLACE_ME"
+        pattern: "temp.fesom..*.nc"
+    cmor_variable: "thetao"
+    model_variable: "temp"

tests/fixtures/config_files.py

@@ -11,3 +11,8 @@ def test_config():
 @pytest.fixture
 def pi_uxarray_config():
     return TEST_ROOT / "configs" / "test_config_pi_uxarray.yaml"
+
+
+@pytest.fixture
+def fesom_2p6_pimesh_esm_tools_config():
+    return TEST_ROOT / "configs" / "test_config_fesom_2p6_pimesh.yaml"

tests/fixtures/example_data/fesom_2p6_pimesh.py

@@ -0,0 +1,43 @@
+"""Example data for the FESOM model."""
+
+import tarfile
+from pathlib import Path
+
+import pytest
+import requests
+
+URL = "https://nextcloud.awi.de/s/7gtFn38ZGifMAfw/download/fesom_2p6_pimesh.tar.gz"
+"""str : URL to download the example data from."""
+
+
+@pytest.fixture
+def download_data(tmp_path_factory):
+    cache_dir = tmp_path_factory.getbasetemp() / "cached_data"
+    cache_dir.mkdir(exist_ok=True)
+    data_path = cache_dir / "fesom_2p6_pimesh.tar.gz"
+
+    if not data_path.exists():
+        response = requests.get(URL)
+        response.raise_for_status()
+        with open(data_path, "wb") as f:
+            f.write(response.content)
+        print(f"Data downloaded: {data_path}.")
+    else:
+        print(f"Using cached data: {data_path}.")
+
+    return data_path
+
+
+@pytest.fixture(scope="session")
+def fesom_2p6_esm_tools_data(download_data):
+    data_dir = Path(download_data).parent / "fesom_2p6_pimesh_esm_tools"
+
+    # Extract only if the directory doesn't already exist
+    if not data_dir.exists():
+        with tarfile.open(download_data, "r:gz") as tar:
+            tar.extractall(data_dir.parent)
+        print(f"Data extracted to: {data_dir}.")
+    else:
+        print(f"Using cached extraction: {data_dir}.")
+
+    return data_dir

tests/fixtures/example_data/pi_uxarray.py

@@ -10,7 +10,7 @@
 """str : URL to download the example data from."""
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def download_data(tmp_path_factory):
     cache_dir = tmp_path_factory.getbasetemp() / "cached_data"
     cache_dir.mkdir(exist_ok=True)

tests/integration/test_fesom_2p6_pimesh_esm_tools.py

@@ -0,0 +1,18 @@
+# import pytest
+import yaml
+
+from pymorize.cmorizer import CMORizer
+from pymorize.logging import logger
+
+
+def test_process(fesom_2p6_pimesh_esm_tools_config, fesom_2p6_pimesh_esm_tools_data):
+    logger.info(f"Processing {fesom_2p6_pimesh_esm_tools_config}")
+    with open(fesom_2p6_pimesh_esm_tools_config, "r") as f:
+        cfg = yaml.safe_load(f)
+    for rule in cfg["rules"]:
+        for input in rule["inputs"]:
+            input["path"] = input["path"].replace(
+                "REPLACE_ME", str(fesom_2p6_pimesh_esm_tools_data)
+            )
+    cmorizer = CMORizer.from_dict(cfg)
+    cmorizer.process()