Merge pull request #421 from MetOffice/fix-unified-model-sync

Add test and try/except to support S3 Glacier object accessibility

forest/__init__.py

@@ -28,7 +28,7 @@
 .. automodule:: forest.services
 
 """
-__version__ = '0.20.5'
+__version__ = '0.20.6'
 
 from .config import *
 from . import (

forest/drivers/unified_model.py

@@ -63,7 +63,13 @@ def __call__(self):
             with forest.db.Database.connect(self.database_path) as database:
                 for path in extra_paths:
                     print("inserting: '{}'".format(path))
-                    database.insert_netcdf(path)
+                    try:
+                        database.insert_netcdf(path)
+                    except OSError as e:
+                        # S3 Glacier objects inaccessible via goofys
+                        print(e)
+                        print(f"skip file: {path}")
+                        continue
             print("finished")
 
     def full_path(self, name):

test/test_drivers_unified_model.py

@@ -9,6 +9,25 @@
 import iris
 
 
+def test_sync_skips_oserror(tmpdir):
+    """Files stored in S3 Glacier are inaccessible via goofys"""
+    database_path = str(tmpdir / "file.db")
+    connection = sqlite3.connect(database_path)
+    connection.close()
+    file_name = str(tmpdir / "file.nc")
+    with open(file_name, "w"):
+        # Simulate NetCDF files stored in S3 Glacier
+        pass
+    settings = {
+        "locator": "database",
+        "pattern": file_name,
+        "directory": str(tmpdir),
+        "database_path": database_path
+    }
+    dataset = forest.drivers.get_dataset("unified_model", settings)
+    dataset.sync()
+
+
 def test_dataset_loader_pattern():
     settings = {
         "pattern": "*.nc",