fix: catalog serialization-deserialization handling NaNs

swiss-seismological-service · Sep 3, 2024 · 75bbbb8 · 75bbbb8
1 parent bf92cc8
commit 75bbbb8
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 28 deletions.
diff --git a/seismostats/catalogs/catalog.py b/seismostats/catalogs/catalog.py
@@ -187,7 +187,7 @@ def from_dict(cls,
 
         for num in numeric_cols:
             if num in df.columns:
-                df[num] = pd.to_numeric(df[num])
+                df[num] = pd.to_numeric(df[num], errors='coerce')
 
         if 'time' in df.columns:
             df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None)

diff --git a/seismostats/catalogs/catalog_templates/quakeml.j2 b/seismostats/catalogs/catalog_templates/quakeml.j2
@@ -14,8 +14,8 @@
                     <author>{{ author }}</author>
                 </creationInfo>
                 <mag>
-                    <value>{{ event.magnitude }}</value>
-                    <uncertainty>{{ event.magnitude_uncertainty }}</uncertainty>
+                    <value>{{ event.magnitude if not event.magnitude is nan else ''}}</value>
+                    <uncertainty>{{ event.magnitude_uncertainty if not event.magnitude_uncertainty is nan else '' }}</uncertainty>
                 </mag>
                 <type>{{ event.magnitude_type }}</type>
                 <originID>{{ event.originid }}</originID>
@@ -28,8 +28,8 @@
                     <author>{{ author }}</author>
                 </creationInfo>
                 <mag>
-                    <value>{{ magnitude.magnitude }}</value>
-                    <uncertainty>{{ magnitude.magnitude_uncertainty }}</uncertainty>
+                    <value>{{ magnitude.magnitude if not magnitude.magnitude is nan else ''}}</value>
+                    <uncertainty>{{ magnitude.magnitude_uncertainty if not magnitude.magnitude_uncertainty is nan else ''}}</uncertainty>
                 </mag>
                 <type>{{ type }}</type>
                 <originID>{{ event.originid }}</originID>
@@ -41,21 +41,21 @@
                     <value>{{ event.time.strftime('%Y-%m-%dT%H:%M:%S.%fZ') }}</value>
                 </time>
                 <longitude>
-                    <value>{{ event.longitude }}</value>
-                    <uncertainty> {{ event.longitude_uncertainty }}</uncertainty>
+                    <value>{{ event.longitude if not event.longitude is nan else ''}}</value>
+                    <uncertainty> {{ event.longitude_uncertainty if not event.longitude_uncertainty is nan else ''}}</uncertainty>
                 </longitude>
                 <latitude>
-                    <value>{{ event.latitude }}</value>
-                    <uncertainty>{{ event.latitude_uncertainty }}</uncertainty>
+                    <value>{{ event.latitude if not event.latitude is nan else ''}}</value>
+                    <uncertainty>{{ event.latitude_uncertainty if not event.latitude_uncertainty is nan else '' }}</uncertainty>
                 </latitude>
                 <evaluationMode>{{ event.evaluationmode }}</evaluationMode>
                 <creationInfo>
                     <agencyID>{{ agencyID }}</agencyID>
                     <author>{{ author }}</author>
                 </creationInfo>
                 <depth>
-                    <value>{{ event.depth }}</value>
-                    <uncertainty>{{ event.depth_uncertainty }}</uncertainty>
+                    <value>{{ event.depth if not event.depth is nan else ''}}</value>
+                    <uncertainty>{{ event.depth_uncertainty if not event.depth_uncertainty is nan else '' }}</uncertainty>
                 </depth>
             </origin>
 

diff --git a/seismostats/utils/__init__.py b/seismostats/utils/__init__.py
@@ -1,22 +1,17 @@
 import functools
+import math
 
 import pandas as pd
-from jinja2 import Template, select_autoescape
-from seismostats.utils.binning import (  # noqa
-    bin_to_precision,
-    get_cum_fmd,
-    get_fmd,
-)
-from seismostats.utils.simulate_distributions import (  # noqa
-    simulate_magnitudes,
-    simulate_magnitudes_binned,
-)
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+from seismostats.utils.binning import (bin_to_precision, get_cum_fmd,  # noqa
+                                       get_fmd)
+from seismostats.utils.coordinates import CoordinateTransformer  # noqa
+from seismostats.utils.coordinates import (bounding_box_to_polygon,  # noqa
+                                           polygon_to_bounding_box)
 from seismostats.utils.filtering import cat_intersect_polygon  # noqa
-from seismostats.utils.coordinates import (  # noqa
-    CoordinateTransformer,
-    bounding_box_to_polygon,
-    polygon_to_bounding_box,
-)
+from seismostats.utils.simulate_distributions import (  # noqa
+    simulate_magnitudes, simulate_magnitudes_binned)
 
 
 def _check_required_cols(df: pd.DataFrame, required_cols: list[str]):
@@ -75,9 +70,19 @@ def wrapper_require(self, *args, **kwargs):
         return decorator_require(_func)
 
 
+def is_nan(value):
+    return isinstance(value, float) and math.isnan(value)
+
+
 def _render_template(data: dict, template_path: str) -> str:
-    with open(template_path) as t:
-        template = Template(t.read(), autoescape=select_autoescape())
+
+    env = Environment(
+        loader=FileSystemLoader('/'),  # Base directory for templates
+        autoescape=select_autoescape()
+    )
+    env.tests['nan'] = is_nan
+
+    template = env.get_template(template_path)
 
     qml = template.render(**data)
     return qml