Lot3 integration with ODS-tools package (#88)

* added imports for lot3 experimentation

* Add lot3 package to CI  (#47)

* Git install lot3 package before testing

Update default oasislmf branch to lot3

update

try py3.10

Revert "try py3.10"

This reverts commit 0999b36.

* Comment out plat testing for the mo

* added configurable source reader to OedSource

* added filtering on location, account and portfolio

* updated get_df_reader calls to use the oed info

* added writing of df engine to exposure_info.json

* added support for different model and exposure data readers

* removed py3.7 support from CI

* removed unused import

* linting

* updated lot3 install in ci

* fixed lot3 install nesting in ci

* Update package ref

* Revert workflows

* Build and test ODM package

* retest

---------

Co-authored-by: Dan Bate <[email protected]>

.github/workflows/test-platform.yml

@@ -14,4 +14,4 @@ jobs:
     secrets: inherit
     with:
       ods_branch: ${{ github.ref }}
-      platform_branch: ${{ github.event_name != 'workflow_dispatch' && 'main-platform1' ||  inputs.platform_branch }}
+      platform_branch: ${{ github.event_name != 'workflow_dispatch' && 'main' ||  inputs.platform_branch }}

.github/workflows/test.yml

@@ -9,16 +9,22 @@ on:
         required: false
 
 jobs:
-  build:
+  build-ods:
     uses: ./.github/workflows/build.yml
     secrets: inherit
     with:
       oed_spec_branch: ${{ inputs.oed_spec_branch }}
 
+  build-odm:
+    uses: OasisLMF/OasisDataManager/.github/workflows/build.yml@fix/release-0.1.0
+    secrets: inherit
+    with:
+      odm_branch: ${{ github.event_name != 'workflow_dispatch' && 'fix/release-0.1.0' ||  inputs.ods_branch }}
+
   test:
     name: Run Pytest
     runs-on: ubuntu-latest
-    needs: build
+    needs: [build-ods, build-odm]
     strategy:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
@@ -31,14 +37,22 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Download package
+    - name: Download package (ODS-tools)
       uses: actions/download-artifact@v3
       with:
         name: bin_package
         path: ${{ github.workspace }}/
 
+    - name: Download package (OasisDataManager)
+      uses: actions/download-artifact@v3
+      with:
+        name: odm_bin_package
+        path: ${{ github.workspace }}/
+
     - name: install package
-      run: pip install ${{ needs.build.outputs.whl_filename }}
+      run: |
+        pip install ${{ needs.build-odm.outputs.whl_filename }}
+        pip install ${{ needs.build-ods.outputs.whl_filename }}
 
     - name: Install test deps
       run: pip install -r tests/requirements.in

ods_tools/oed/common.py

@@ -148,7 +148,8 @@ def fill_empty(df, columns, value):
     if isinstance(columns, str):
         columns = [columns]
     for column in columns:
-        if df[column].dtypes.name == 'category' and value not in {None, np.nan}.union(df[column].cat.categories):
+        dtype = getattr(df[column], "dtypes", getattr(df[column], "dtype", None))
+        if dtype.name == 'category' and value not in {None, np.nan}.union(df[column].cat.categories):
             df[column] = df[column].cat.add_categories(value)
         df.loc[df[column].isin(BLANK_VALUES), column] = value
 

ods_tools/oed/exposure.py

@@ -6,6 +6,7 @@
 """
 
 import json
+from copy import deepcopy
 import logging
 from packaging import version
 from pathlib import Path
@@ -39,7 +40,12 @@ def __init__(self,
                  check_oed=False,
                  use_field=False,
                  validation_config=None,
-                 working_dir=None):
+                 working_dir=None,
+                 location_numbers=None,
+                 account_numbers=None,
+                 portfolio_numbers=None,
+                 base_df_engine=None,
+                 exposure_df_engine=None):
         """
         Create an OED object,
         each input can be the object itself or  information that will be used to create the object
@@ -54,14 +60,69 @@ def __init__(self,
             reporting_currency (str): currency to convert
             check_oed (bool): check if OED files are valid or not
             use_field (bool): if true column name are converted to OED field name on load
+            location_numbers (list[str]): A list of location numbers to filter the input data by
+            account_numbers (list[str]): A list of account numbers to filter the input data by
+            portfolio_numbers (list[str]): A list of portfolio numbers to filter the input data by
+            base_df_engine (Union[str, InputReaderConfig]): The default engine to use when loading dataframes
+            exposure_df_engine (Union[str, InputReaderConfig]):
+                The exposure specific engine to use when loading dataframes
         """
         self.use_field = use_field
         self.oed_schema = OedSchema.from_oed_schema_info(oed_schema_info)
+        df_engine = (
+            exposure_df_engine or
+            base_df_engine or
+            'oasis_data_manager.df_reader.reader.OasisPandasReader'
+        )
+
+        def filter_col_in(column, values):
+            def fn(df):
+                if column not in df.columns or not values:
+                    return df
+
+                return df[df[column].isin(values)]
+            return fn
+
+        loc_filters = [
+            filter_col_in("LocNumber", location_numbers),
+            filter_col_in("AccNumber", account_numbers),
+            filter_col_in("PortNumber", portfolio_numbers),
+        ]
+        self.location = OedSource.from_oed_info(
+            exposure=self,
+            oed_type='Loc',
+            oed_info=self.resolve_oed_info(location, df_engine),
+            filters=loc_filters,
+        )
 
-        self.location = OedSource.from_oed_info(exposure=self, oed_type='Loc', oed_info=location)
-        self.account = OedSource.from_oed_info(exposure=self, oed_type='Acc', oed_info=account)
-        self.ri_info = OedSource.from_oed_info(exposure=self, oed_type='ReinsInfo', oed_info=ri_info)
-        self.ri_scope = OedSource.from_oed_info(exposure=self, oed_type='ReinsScope', oed_info=ri_scope)
+        acc_filters = [
+            filter_col_in("AccNumber", account_numbers),
+            filter_col_in("PortNumber", portfolio_numbers),
+        ]
+        self.account = OedSource.from_oed_info(
+            exposure=self,
+            oed_type='Acc',
+            oed_info=self.resolve_oed_info(account, df_engine),
+            filters=acc_filters,
+        )
+
+        self.ri_info = OedSource.from_oed_info(
+            exposure=self,
+            oed_type='ReinsInfo',
+            oed_info=self.resolve_oed_info(ri_info, df_engine),
+        )
+
+        ri_scope_filters = [
+            filter_col_in("LocNumber", location_numbers),
+            filter_col_in("AccNumber", account_numbers),
+            filter_col_in("PortNumber", portfolio_numbers),
+        ]
+        self.ri_scope = OedSource.from_oed_info(
+            exposure=self,
+            oed_type='ReinsScope',
+            oed_info=self.resolve_oed_info(ri_scope, df_engine),
+            filters=ri_scope_filters,
+        )
 
         self.currency_conversion = create_currency_rates(currency_conversion)
 
@@ -77,6 +138,29 @@ def __init__(self,
         if check_oed:
             self.check()
 
+    @classmethod
+    def resolve_oed_info(cls, oed_info, df_engine):
+        if isinstance(oed_info, (str, Path)):
+            return {
+                "cur_version_name": "curr",
+                "sources": {
+                    "curr": {
+                        "source_type": "filepath",
+                        "filepath": oed_info,
+                        "read_param": {},
+                        "engine": df_engine
+                    }
+                }
+            }
+        elif isinstance(oed_info, dict):
+            if "sources" in oed_info:
+                oed_info = deepcopy(oed_info)
+                for k in oed_info["sources"]:
+                    oed_info["sources"][k].setdefault("engine", df_engine)
+                return oed_info
+
+        return oed_info
+
     @classmethod
     def from_config(cls, config_fp, **kwargs):
         """
@@ -220,9 +304,11 @@ def save(self, path, version_name=None, compression=None, save_config=False, unk
 
             filepath = filepath.with_suffix(PANDAS_COMPRESSION_MAP[compression])
 
-            oed_source.save(saved_version_name + '_' + f'{compression}',
-                            {'source_type': 'filepath', 'filepath': filepath, 'extension': compression},
-                            unknown_columns=unknown_columns)
+            new_info = {'source_type': 'filepath', 'filepath': filepath, 'extension': compression}
+            if "engine" in oed_source.sources[oed_source.cur_version_name]:
+                new_info["engine"] = oed_source.sources[oed_source.cur_version_name]["engine"]
+
+            oed_source.save(saved_version_name + '_' + f'{compression}', new_info, unknown_columns=unknown_columns)
         if save_config:
             self.save_config(Path(path, self.DEFAULT_EXPOSURE_CONFIG_NAME))