Release 3.1.3 (#74)

* Set package to version 3.1.3 * Fix/forex case error (#70) * fix case to detect the correct column when doing a currency conversion * add list option to create currency conversion object * Update model_settings_schema.json adding event set optional dependency with valid footprint ids (#69) Allow model vendors to set dependency between event set and footprint files * Feature/versioning (#66) * CI improvement, select OED branch * check if succeeding tests on push * testing success on push * added tests * added check that versioning is present in schema * compression argument not required anymore * added test of successful versioning * commented code * improved logging and checks * code quality * review: now no need for exact version to be passed * check columns OccupancyCode and ConstructionCode exist * added warning if desired version is < than supported * removed unnecessary warnings * refactored using version from packaging library * setup.py pointing to ODS_OpenExposureData with versioning info * Update setup.py to released spec 3.1.0 --------- Co-authored-by: sambles <[email protected]> * Feature/peril filter (#73) * add peril_filter function * add numba as requirement * Update changelog * Fix changelog --------- Co-authored-by: awsbuild <[email protected]> Co-authored-by: Stephane Struzik <[email protected]> Co-authored-by: fl-ndaq <[email protected]> Co-authored-by: Nicola Cerutti <[email protected]>
OasisLMF · Nov 10, 2023 · a9795e0 · a9795e0
1 parent b6e453d
commit a9795e0
Show file tree

Hide file tree

Showing 15 changed files with 515 additions and 75 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -2,6 +2,10 @@ name: ODS-tools Build
 
 on:
   workflow_dispatch:
+    inputs:
+      oed_spec_branch:
+        description: 'Branch of OED spec'
+        required: false
   workflow_call:
     inputs:
       ods_branch:
@@ -12,6 +16,10 @@ on:
         description: 'Test unreleased OED spec'
         required: false
         type: string
+      oed_spec_branch:
+        description: 'Branch of OED spec'
+        required: false
+        type: string
     outputs:
       src_filename:
         description: "Source Package filename"
@@ -21,7 +29,16 @@ on:
         value: ${{ jobs.build.outputs.whl_filename }}
 
 jobs:
+  oed_spec:
+    if: inputs.oed_spec_branch != ''
+    uses:  OasisLMF/ODS_OpenExposureData/.github/workflows/build.yml@develop
+    secrets: inherit
+    with:
+      ods_branch: ${{ inputs.oed_spec_branch }}
+
   build:
+    if: ${{ ! failure() || ! cancelled() }}
+    needs: [oed_spec]
     name: Build OpenData Package
     runs-on: ubuntu-latest
     outputs:
@@ -62,17 +79,17 @@ jobs:
         python setup.py bdist_wheel
 
     - name: Download OED spec
-      if: inputs.oed_spec_json != ''
+      if: inputs.oed_spec_json != '' || inputs.oed_spec_branch != ''
       uses: actions/download-artifact@v3
       with:
         name: extracted_spec
         path: ${{ github.workspace }}/
 
     # Testing only - not for release
-    - name: Build package (from OED schema file)
-      if: inputs.oed_spec_json != ''
+    - name: Buildpackage (from OED schema file)
+      if: inputs.oed_spec_json != '' || inputs.oed_spec_branch != ''
       run: |
-        python setup.py bdist_wheel install "--local-oed-spec=${{ github.workspace }}/${{ inputs.oed_spec_json }}"
+        python setup.py bdist_wheel install "--local-oed-spec=${{ github.workspace }}/OpenExposureData_Spec.json"
 
     - name: Build Output
       id: build_package
@@ -94,4 +111,4 @@ jobs:
       with:
         name: bin_package
         path: ./dist/${{ steps.build_package.outputs.wheel }}
-        retention-days: 5
+        retention-days: 5
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,14 +1,19 @@
 name: ODS-tools Testing
 
-
 on:
   push:
   workflow_dispatch:
+    inputs:
+      oed_spec_branch:
+        description: 'Branch of OED spec'
+        required: false
 
 jobs:
   build:
     uses: ./.github/workflows/build.yml
     secrets: inherit
+    with:
+      oed_spec_branch: ${{ inputs.oed_spec_branch }}
 
   test:
     name: Run Pytest
@@ -39,4 +44,4 @@ jobs:
       run: pip install -r tests/requirements.in
 
     - name: Run tests
-      run: pytest -v
+      run: pytest -v
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,14 @@
 ODS_Tools Changelog
 ===================
 
+`3.1.3`_
+ ---------
+* [#64](https://github.com/OasisLMF/ODS_Tools/pull/66) - Backward compatibility when adding new codes in OED
+* [#68](https://github.com/OasisLMF/ODS_Tools/pull/69) - Define relationships between event and footprint sets
+* [#70](https://github.com/OasisLMF/ODS_Tools/pull/70) - Fix/forex case error
+* [#73](https://github.com/OasisLMF/ODS_Tools/pull/73) - Feature/peril filter
+.. _`3.1.3`:  https://github.com/OasisLMF/ODS_Tools/compare/3.1.2...3.1.3
+
 `3.1.2`_
  ---------
 * [#53](https://github.com/OasisLMF/ODS_Tools/pull/53) - Release 3.1.1 (staging)

diff --git a/ods_tools/__init__.py b/ods_tools/__init__.py
@@ -1,4 +1,4 @@
-__version__ = '3.1.2'
+__version__ = '3.1.3'
 
 import logging
 

diff --git a/ods_tools/data/model_settings_schema.json b/ods_tools/data/model_settings_schema.json
@@ -133,6 +133,15 @@
                                  "minLength":1
                               }
                            },
+                           "valid_footprint_ids":{
+                              "type":"array",
+                              "title":"Supported footprint files",
+                              "description":"An optional list of viable footprint file ids to use with this event set",
+                              "items":{
+                                 "type":"string",
+                                 "minLength":1
+                              }
+                           },
                            "valid_perspectives":{
                               "type":"array",
                               "title":"Supported loss perspectives ",

diff --git a/ods_tools/main.py b/ods_tools/main.py
@@ -9,6 +9,7 @@
 
 import argparse
 import logging
+import os
 
 from ods_tools import logger
 from ods_tools.oed import (
@@ -57,11 +58,37 @@ def check(**kwargs):
 
 
 def convert(**kwargs):
-    """Convert exposure data to an other format (ex: csv to parquet)"""
-    path = kwargs.pop('output_dir', None) or kwargs.get('oed_dir', None)
+    """Convert exposure data to an other format (ex: csv to parquet) or version (ex: 3.0 to 2.2)"""
+    path = kwargs.pop('output_dir', None)
+
+    if not kwargs.get("compression") and not kwargs.get("version"):
+        raise OdsException("either --compression or --version must be provided")
+
+    if kwargs.get("config_json"):
+        if not path:
+            path = os.path.dirname(kwargs.get("config_json"))
+    elif kwargs.get("oed_dir"):
+        path = kwargs.get("oed_dir")
+    elif kwargs.get("location"):
+        if not path:
+            raise OdsException("output_dir must be provided when location is provided as single file")
+
     if not path:
-        raise OdsException('--output-dir or --oed-dir need to be provided to perform convert')
+        kwargs["oed_dir"] = path = os.getcwd()
+
     oed_exposure = get_oed_exposure(**extract_exposure_args(kwargs))
+
+    version = kwargs.pop("version", None)
+    if version:
+        logger.info(f"Converting to version {version}.")  # Log the conversion version
+        try:
+            oed_exposure.to_version(version)
+            kwargs["version_name"] = version.replace(".", "-")
+        except OdsException as e:
+            logger.error("Conversion failed:")
+            logger.error(e)
+
+    logger.info(f"Saving to: {path}")
     oed_exposure.save(path=path, **kwargs)
 
 
@@ -95,7 +122,7 @@ def add_exposure_data_args(command):
 
 
 convert_description = """
-convert OED files to an other format
+convert OED files to an other format or version
 """
 
 command_parser = main_parser.add_subparsers(help='command [convert]', dest='command', required=True)
@@ -104,10 +131,11 @@ def add_exposure_data_args(command):
 add_exposure_data_args(convert_command)
 convert_command.add_argument('--check-oed', help='if True, OED file will be checked before convertion', default=False)
 convert_command.add_argument('--output-dir', help='path of the output directory', required=False)
-convert_command.add_argument('-c', '--compression', help='compression to use (ex: parquet, zip, gzip, csv,...)', required=True)
+convert_command.add_argument('-c', '--compression', help='compression to use (ex: parquet, zip, gzip, csv,...)', required=False)
 convert_command.add_argument('--save-config', help='if True, OED config file will be save in the --path directory', default=False)
 convert_command.add_argument('-v', '--logging-level', help='logging level (debug:10, info:20, warning:30, error:40, critical:50)',
                              default=30, type=int)
+convert_command.add_argument('--version', help='specific OED version to use in the conversion', default=None, type=str)
 
 check_description = """
 check exposure data.

diff --git a/ods_tools/oed/__init__.py b/ods_tools/oed/__init__.py
@@ -5,13 +5,13 @@
 from .common import (
     OdsException, PANDAS_COMPRESSION_MAP, PANDAS_DEFAULT_NULL_VALUES, USUAL_FILE_NAME, OED_TYPE_TO_NAME,
     OED_NAME_TO_TYPE, OED_IDENTIFIER_FIELDS, VALIDATOR_ON_ERROR_ACTION, DEFAULT_VALIDATION_CONFIG, OED_PERIL_COLUMNS, fill_empty,
-    UnknownColumnSaveOption
+    UnknownColumnSaveOption, BLANK_VALUES
 )
 
 
 __all__ = [
     'OedExposure', 'OedSchema', 'OedSource', 'ModelSettingSchema', 'AnalysisSettingSchema',
     'OdsException', 'PANDAS_COMPRESSION_MAP', 'PANDAS_DEFAULT_NULL_VALUES', 'USUAL_FILE_NAME', 'OED_TYPE_TO_NAME',
     'OED_NAME_TO_TYPE', 'OED_IDENTIFIER_FIELDS', 'VALIDATOR_ON_ERROR_ACTION', 'DEFAULT_VALIDATION_CONFIG', 'OED_PERIL_COLUMNS', 'fill_empty',
-    'UnknownColumnSaveOption'
+    'UnknownColumnSaveOption', 'BLANK_VALUES'
 ]
diff --git a/ods_tools/oed/common.py b/ods_tools/oed/common.py
@@ -25,6 +25,52 @@ def is_relative(filepath):
     return not (all([url_parsed.scheme, url_parsed.netloc]) or Path(filepath).is_absolute())
 
 
+try:
+    from functools import cached_property
+except ImportError:  # support for python < 3.8
+    _missing = object()
+
+    class cached_property(object):
+        """A decorator that converts a function into a lazy property.  The
+        function wrapped is called the first time to retrieve the result
+        and then that calculated result is used the next time you access
+        the value::
+
+            class Foo(object):
+
+                @cached_property
+                def foo(self):
+                    # calculate something important here
+                    return 42
+
+        The class has to have a `__dict__` in order for this property to
+        work.
+        """
+
+        # implementation detail: this property is implemented as non-data
+        # descriptor.  non-data descriptors are only invoked if there is
+        # no entry with the same name in the instance's __dict__.
+        # this allows us to completely get rid of the access function call
+        # overhead.  If one choses to invoke __get__ by hand the property
+        # will still work as expected because the lookup logic is replicated
+        # in __get__ for manual invocation.
+
+        def __init__(self, func, name=None, doc=None):
+            self.__name__ = name or func.__name__
+            self.__module__ = func.__module__
+            self.__doc__ = doc or func.__doc__
+            self.func = func
+
+        def __get__(self, obj, type=None):
+            if obj is None:
+                return self
+            value = obj.__dict__.get(self.__name__, _missing)
+            if value is _missing:
+                value = self.func(obj)
+                obj.__dict__[self.__name__] = value
+            return value
+
+
 # PANDAS_COMPRESSION_MAP is also used to order the preferred input format in ExposureData.from_dir
 PANDAS_COMPRESSION_MAP = {
     'parquet': '.parquet',

diff --git a/ods_tools/oed/exposure.py b/ods_tools/oed/exposure.py
@@ -6,6 +6,8 @@
 """
 
 import json
+import logging
+from packaging import version
 from pathlib import Path
 
 from .common import (PANDAS_COMPRESSION_MAP,
@@ -16,10 +18,12 @@
 from .validator import Validator
 from .forex import create_currency_rates
 
+logger = logging.getLogger(__name__)
+
 
 class OedExposure:
     """
-    Object grouping all the OED files related to the exposure Data (location, acoount, ri_info, ri_scope)
+    Object grouping all the OED files related to the exposure Data (location, account, ri_info, ri_scope)
     and the OED schema to follow
     """
     DEFAULT_EXPOSURE_CONFIG_NAME = 'exposure_info.json'
@@ -101,16 +105,27 @@ def find_fp(names):
                     if Path(oed_dir, name).with_suffix(extension).is_file():
                         return Path(oed_dir, name).with_suffix(extension)
 
+        necessary_files = ["location"]
+        files_found = {file: False for file in necessary_files}
+
         if Path(oed_dir, cls.DEFAULT_EXPOSURE_CONFIG_NAME).is_file():
             return cls.from_config(Path(oed_dir, cls.DEFAULT_EXPOSURE_CONFIG_NAME), **kwargs)
         else:
             config = {}
             for attr, filenames in USUAL_FILE_NAME.items():
-                config[attr] = find_fp(filenames)
+                file_path = find_fp(filenames)
+                if file_path and attr in files_found:
+                    files_found[attr] = True
+                config[attr] = file_path
 
             if Path(oed_dir, OedSchema.DEFAULT_ODS_SCHEMA_FILE).is_file():
                 config['oed_schema_info'] = Path(oed_dir, OedSchema.DEFAULT_ODS_SCHEMA_FILE)
             kwargs['working_dir'] = oed_dir
+
+            missing_files = [file for file, found in files_found.items() if not found]
+            if missing_files:
+                raise FileNotFoundError(f"Files not found in current path ({oed_dir}): {', '.join(missing_files)}")
+
             return cls(**{**config, **kwargs})
 
     @property
@@ -229,3 +244,71 @@ def check(self, validation_config=None):
             validation_config = self.validation_config
         validator = Validator(self)
         return validator(validation_config)
+
+    def to_version(self, to_version):
+        """
+        goes through location to convert columns to specific version.
+        Right now it works for OccupancyCode and ConstructionCode.
+        (please note that it only supports minor version changes in "major.minor" format, e.g. 3.2, 7.4, etc.)
+
+        Args:
+            version (str): specific version to roll to
+
+        Returns:
+            itself (OedExposure): updated object
+        """
+        def strip_version(version_string):
+            parsed_version = version.parse(version_string)
+            return f'{parsed_version.release[0]}.{parsed_version.release[1]}'
+
+        # Parse version string
+        try:
+            target_version = strip_version(to_version)
+        except version.InvalidVersion:
+            raise ValueError(f"Invalid version: {to_version}")
+
+        # Select which conversions to apply
+        conversions = sorted(
+            [
+                ver
+                for ver in self.oed_schema.schema["versioning"].keys()
+                if version.parse(ver) >= version.parse(target_version)
+            ],
+            key=lambda x: version.parse(x),
+            reverse=True,
+        )
+
+        # Check for the existence of OccupancyCode and ConstructionCode columns
+        has_occupancy_code = "OccupancyCode" in self.location.dataframe.columns
+        has_construction_code = "ConstructionCode" in self.location.dataframe.columns
+
+        for ver in conversions:
+            # Create a dictionary for each category
+            replace_dict_occupancy = {}
+            replace_dict_construction = {}
+
+            for rule in self.oed_schema.schema["versioning"][ver]:
+                if rule["Category"] == "Occupancy":
+                    replace_dict_occupancy[rule["New code"]] = rule["Fallback"]
+                elif rule["Category"] == "Construction":
+                    replace_dict_construction[rule["New code"]] = rule["Fallback"]
+
+            # Replace and log changes for OccupancyCode
+            if has_occupancy_code:
+                for key, value in replace_dict_occupancy.items():
+                    # Check done in advance to log what is being changed
+                    changes = self.location.dataframe["OccupancyCode"][self.location.dataframe["OccupancyCode"] == key].count()
+                    if changes:
+                        self.location.dataframe["OccupancyCode"].replace({key: value}, inplace=True)
+                        logger.info(f"{key} -> {value}: {changes} occurrences in OccupancyCode.")
+
+            # Replace and log changes for ConstructionCode
+            if has_construction_code:
+                for key, value in replace_dict_construction.items():
+                    # Check done in advance to log what is being changed
+                    changes = self.location.dataframe["ConstructionCode"][self.location.dataframe["ConstructionCode"] == key].count()
+                    if changes:
+                        self.location.dataframe["ConstructionCode"].replace({key: value}, inplace=True)
+                        logger.info(f"{key} -> {value}: {changes} occurrences in ConstructionCode.")
+
+        return self  # Return the updated object