diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e553699..2ea0d902 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,10 @@ name: ODS-tools Build on: workflow_dispatch: + inputs: + oed_spec_branch: + description: 'Branch of OED spec' + required: false workflow_call: inputs: ods_branch: @@ -12,6 +16,10 @@ on: description: 'Test unreleased OED spec' required: false type: string + oed_spec_branch: + description: 'Branch of OED spec' + required: false + type: string outputs: src_filename: description: "Source Package filename" @@ -21,7 +29,16 @@ on: value: ${{ jobs.build.outputs.whl_filename }} jobs: + oed_spec: + if: inputs.oed_spec_branch != '' + uses: OasisLMF/ODS_OpenExposureData/.github/workflows/build.yml@develop + secrets: inherit + with: + ods_branch: ${{ inputs.oed_spec_branch }} + build: + if: ${{ ! failure() || ! cancelled() }} + needs: [oed_spec] name: Build OpenData Package runs-on: ubuntu-latest outputs: @@ -62,17 +79,17 @@ jobs: python setup.py bdist_wheel - name: Download OED spec - if: inputs.oed_spec_json != '' + if: inputs.oed_spec_json != '' || inputs.oed_spec_branch != '' uses: actions/download-artifact@v3 with: name: extracted_spec path: ${{ github.workspace }}/ # Testing only - not for release - - name: Build package (from OED schema file) - if: inputs.oed_spec_json != '' + - name: Buildpackage (from OED schema file) + if: inputs.oed_spec_json != '' || inputs.oed_spec_branch != '' run: | - python setup.py bdist_wheel install "--local-oed-spec=${{ github.workspace }}/${{ inputs.oed_spec_json }}" + python setup.py bdist_wheel install "--local-oed-spec=${{ github.workspace }}/OpenExposureData_Spec.json" - name: Build Output id: build_package @@ -94,4 +111,4 @@ jobs: with: name: bin_package path: ./dist/${{ steps.build_package.outputs.wheel }} - retention-days: 5 + retention-days: 5 \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e1b78fe..ae440016 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,14 +1,19 @@ name: ODS-tools Testing - on: push: workflow_dispatch: + inputs: + oed_spec_branch: + description: 'Branch of OED spec' + required: false jobs: build: uses: ./.github/workflows/build.yml secrets: inherit + with: + oed_spec_branch: ${{ inputs.oed_spec_branch }} test: name: Run Pytest @@ -39,4 +44,4 @@ jobs: run: pip install -r tests/requirements.in - name: Run tests - run: pytest -v + run: pytest -v \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4f29b134..fe6e0a58 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ ODS_Tools Changelog =================== +`3.1.3`_ + --------- +* [#64](https://github.com/OasisLMF/ODS_Tools/pull/66) - Backward compatibility when adding new codes in OED +* [#68](https://github.com/OasisLMF/ODS_Tools/pull/69) - Define relationships between event and footprint sets +* [#70](https://github.com/OasisLMF/ODS_Tools/pull/70) - Fix/forex case error +* [#73](https://github.com/OasisLMF/ODS_Tools/pull/73) - Feature/peril filter +.. _`3.1.3`: https://github.com/OasisLMF/ODS_Tools/compare/3.1.2...3.1.3 + `3.1.2`_ --------- * [#53](https://github.com/OasisLMF/ODS_Tools/pull/53) - Release 3.1.1 (staging) diff --git a/ods_tools/__init__.py b/ods_tools/__init__.py index d8d7f33d..9c88df38 100644 --- a/ods_tools/__init__.py +++ b/ods_tools/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.1.2' +__version__ = '3.1.3' import logging diff --git a/ods_tools/data/model_settings_schema.json b/ods_tools/data/model_settings_schema.json index 70e9381f..1dd6e035 100644 --- a/ods_tools/data/model_settings_schema.json +++ b/ods_tools/data/model_settings_schema.json @@ -133,6 +133,15 @@ "minLength":1 } }, + "valid_footprint_ids":{ + "type":"array", + "title":"Supported footprint files", + "description":"An optional list of viable footprint file ids to use with this event set", + "items":{ + "type":"string", + "minLength":1 + } + }, "valid_perspectives":{ "type":"array", "title":"Supported loss perspectives ", diff --git a/ods_tools/main.py b/ods_tools/main.py index 690ba7f2..fdadfb09 100644 --- a/ods_tools/main.py +++ b/ods_tools/main.py @@ -9,6 +9,7 @@ import argparse import logging +import os from ods_tools import logger from ods_tools.oed import ( @@ -57,11 +58,37 @@ def check(**kwargs): def convert(**kwargs): - """Convert exposure data to an other format (ex: csv to parquet)""" - path = kwargs.pop('output_dir', None) or kwargs.get('oed_dir', None) + """Convert exposure data to an other format (ex: csv to parquet) or version (ex: 3.0 to 2.2)""" + path = kwargs.pop('output_dir', None) + + if not kwargs.get("compression") and not kwargs.get("version"): + raise OdsException("either --compression or --version must be provided") + + if kwargs.get("config_json"): + if not path: + path = os.path.dirname(kwargs.get("config_json")) + elif kwargs.get("oed_dir"): + path = kwargs.get("oed_dir") + elif kwargs.get("location"): + if not path: + raise OdsException("output_dir must be provided when location is provided as single file") + if not path: - raise OdsException('--output-dir or --oed-dir need to be provided to perform convert') + kwargs["oed_dir"] = path = os.getcwd() + oed_exposure = get_oed_exposure(**extract_exposure_args(kwargs)) + + version = kwargs.pop("version", None) + if version: + logger.info(f"Converting to version {version}.") # Log the conversion version + try: + oed_exposure.to_version(version) + kwargs["version_name"] = version.replace(".", "-") + except OdsException as e: + logger.error("Conversion failed:") + logger.error(e) + + logger.info(f"Saving to: {path}") oed_exposure.save(path=path, **kwargs) @@ -95,7 +122,7 @@ def add_exposure_data_args(command): convert_description = """ -convert OED files to an other format +convert OED files to an other format or version """ command_parser = main_parser.add_subparsers(help='command [convert]', dest='command', required=True) @@ -104,10 +131,11 @@ def add_exposure_data_args(command): add_exposure_data_args(convert_command) convert_command.add_argument('--check-oed', help='if True, OED file will be checked before convertion', default=False) convert_command.add_argument('--output-dir', help='path of the output directory', required=False) -convert_command.add_argument('-c', '--compression', help='compression to use (ex: parquet, zip, gzip, csv,...)', required=True) +convert_command.add_argument('-c', '--compression', help='compression to use (ex: parquet, zip, gzip, csv,...)', required=False) convert_command.add_argument('--save-config', help='if True, OED config file will be save in the --path directory', default=False) convert_command.add_argument('-v', '--logging-level', help='logging level (debug:10, info:20, warning:30, error:40, critical:50)', default=30, type=int) +convert_command.add_argument('--version', help='specific OED version to use in the conversion', default=None, type=str) check_description = """ check exposure data. diff --git a/ods_tools/oed/__init__.py b/ods_tools/oed/__init__.py index faf93aa3..996bf014 100644 --- a/ods_tools/oed/__init__.py +++ b/ods_tools/oed/__init__.py @@ -5,7 +5,7 @@ from .common import ( OdsException, PANDAS_COMPRESSION_MAP, PANDAS_DEFAULT_NULL_VALUES, USUAL_FILE_NAME, OED_TYPE_TO_NAME, OED_NAME_TO_TYPE, OED_IDENTIFIER_FIELDS, VALIDATOR_ON_ERROR_ACTION, DEFAULT_VALIDATION_CONFIG, OED_PERIL_COLUMNS, fill_empty, - UnknownColumnSaveOption + UnknownColumnSaveOption, BLANK_VALUES ) @@ -13,5 +13,5 @@ 'OedExposure', 'OedSchema', 'OedSource', 'ModelSettingSchema', 'AnalysisSettingSchema', 'OdsException', 'PANDAS_COMPRESSION_MAP', 'PANDAS_DEFAULT_NULL_VALUES', 'USUAL_FILE_NAME', 'OED_TYPE_TO_NAME', 'OED_NAME_TO_TYPE', 'OED_IDENTIFIER_FIELDS', 'VALIDATOR_ON_ERROR_ACTION', 'DEFAULT_VALIDATION_CONFIG', 'OED_PERIL_COLUMNS', 'fill_empty', - 'UnknownColumnSaveOption' + 'UnknownColumnSaveOption', 'BLANK_VALUES' ] diff --git a/ods_tools/oed/common.py b/ods_tools/oed/common.py index 98aa5e93..4cf02008 100644 --- a/ods_tools/oed/common.py +++ b/ods_tools/oed/common.py @@ -25,6 +25,52 @@ def is_relative(filepath): return not (all([url_parsed.scheme, url_parsed.netloc]) or Path(filepath).is_absolute()) +try: + from functools import cached_property +except ImportError: # support for python < 3.8 + _missing = object() + + class cached_property(object): + """A decorator that converts a function into a lazy property. The + function wrapped is called the first time to retrieve the result + and then that calculated result is used the next time you access + the value:: + + class Foo(object): + + @cached_property + def foo(self): + # calculate something important here + return 42 + + The class has to have a `__dict__` in order for this property to + work. + """ + + # implementation detail: this property is implemented as non-data + # descriptor. non-data descriptors are only invoked if there is + # no entry with the same name in the instance's __dict__. + # this allows us to completely get rid of the access function call + # overhead. If one choses to invoke __get__ by hand the property + # will still work as expected because the lookup logic is replicated + # in __get__ for manual invocation. + + def __init__(self, func, name=None, doc=None): + self.__name__ = name or func.__name__ + self.__module__ = func.__module__ + self.__doc__ = doc or func.__doc__ + self.func = func + + def __get__(self, obj, type=None): + if obj is None: + return self + value = obj.__dict__.get(self.__name__, _missing) + if value is _missing: + value = self.func(obj) + obj.__dict__[self.__name__] = value + return value + + # PANDAS_COMPRESSION_MAP is also used to order the preferred input format in ExposureData.from_dir PANDAS_COMPRESSION_MAP = { 'parquet': '.parquet', diff --git a/ods_tools/oed/exposure.py b/ods_tools/oed/exposure.py index 3bdcf7fd..a1bfd901 100644 --- a/ods_tools/oed/exposure.py +++ b/ods_tools/oed/exposure.py @@ -6,6 +6,8 @@ """ import json +import logging +from packaging import version from pathlib import Path from .common import (PANDAS_COMPRESSION_MAP, @@ -16,10 +18,12 @@ from .validator import Validator from .forex import create_currency_rates +logger = logging.getLogger(__name__) + class OedExposure: """ - Object grouping all the OED files related to the exposure Data (location, acoount, ri_info, ri_scope) + Object grouping all the OED files related to the exposure Data (location, account, ri_info, ri_scope) and the OED schema to follow """ DEFAULT_EXPOSURE_CONFIG_NAME = 'exposure_info.json' @@ -101,16 +105,27 @@ def find_fp(names): if Path(oed_dir, name).with_suffix(extension).is_file(): return Path(oed_dir, name).with_suffix(extension) + necessary_files = ["location"] + files_found = {file: False for file in necessary_files} + if Path(oed_dir, cls.DEFAULT_EXPOSURE_CONFIG_NAME).is_file(): return cls.from_config(Path(oed_dir, cls.DEFAULT_EXPOSURE_CONFIG_NAME), **kwargs) else: config = {} for attr, filenames in USUAL_FILE_NAME.items(): - config[attr] = find_fp(filenames) + file_path = find_fp(filenames) + if file_path and attr in files_found: + files_found[attr] = True + config[attr] = file_path if Path(oed_dir, OedSchema.DEFAULT_ODS_SCHEMA_FILE).is_file(): config['oed_schema_info'] = Path(oed_dir, OedSchema.DEFAULT_ODS_SCHEMA_FILE) kwargs['working_dir'] = oed_dir + + missing_files = [file for file, found in files_found.items() if not found] + if missing_files: + raise FileNotFoundError(f"Files not found in current path ({oed_dir}): {', '.join(missing_files)}") + return cls(**{**config, **kwargs}) @property @@ -229,3 +244,71 @@ def check(self, validation_config=None): validation_config = self.validation_config validator = Validator(self) return validator(validation_config) + + def to_version(self, to_version): + """ + goes through location to convert columns to specific version. + Right now it works for OccupancyCode and ConstructionCode. + (please note that it only supports minor version changes in "major.minor" format, e.g. 3.2, 7.4, etc.) + + Args: + version (str): specific version to roll to + + Returns: + itself (OedExposure): updated object + """ + def strip_version(version_string): + parsed_version = version.parse(version_string) + return f'{parsed_version.release[0]}.{parsed_version.release[1]}' + + # Parse version string + try: + target_version = strip_version(to_version) + except version.InvalidVersion: + raise ValueError(f"Invalid version: {to_version}") + + # Select which conversions to apply + conversions = sorted( + [ + ver + for ver in self.oed_schema.schema["versioning"].keys() + if version.parse(ver) >= version.parse(target_version) + ], + key=lambda x: version.parse(x), + reverse=True, + ) + + # Check for the existence of OccupancyCode and ConstructionCode columns + has_occupancy_code = "OccupancyCode" in self.location.dataframe.columns + has_construction_code = "ConstructionCode" in self.location.dataframe.columns + + for ver in conversions: + # Create a dictionary for each category + replace_dict_occupancy = {} + replace_dict_construction = {} + + for rule in self.oed_schema.schema["versioning"][ver]: + if rule["Category"] == "Occupancy": + replace_dict_occupancy[rule["New code"]] = rule["Fallback"] + elif rule["Category"] == "Construction": + replace_dict_construction[rule["New code"]] = rule["Fallback"] + + # Replace and log changes for OccupancyCode + if has_occupancy_code: + for key, value in replace_dict_occupancy.items(): + # Check done in advance to log what is being changed + changes = self.location.dataframe["OccupancyCode"][self.location.dataframe["OccupancyCode"] == key].count() + if changes: + self.location.dataframe["OccupancyCode"].replace({key: value}, inplace=True) + logger.info(f"{key} -> {value}: {changes} occurrences in OccupancyCode.") + + # Replace and log changes for ConstructionCode + if has_construction_code: + for key, value in replace_dict_construction.items(): + # Check done in advance to log what is being changed + changes = self.location.dataframe["ConstructionCode"][self.location.dataframe["ConstructionCode"] == key].count() + if changes: + self.location.dataframe["ConstructionCode"].replace({key: value}, inplace=True) + logger.info(f"{key} -> {value}: {changes} occurrences in ConstructionCode.") + + return self # Return the updated object diff --git a/ods_tools/oed/forex.py b/ods_tools/oed/forex.py index 7fa2c7d6..48272e60 100644 --- a/ods_tools/oed/forex.py +++ b/ods_tools/oed/forex.py @@ -192,8 +192,10 @@ def get_path(name): elif currency_conversion.get("source_type") == 'parquet': return DictBasedCurrencyRates.from_parquet(get_path('file_path'), **currency_conversion.get("read_parameters", {})) - elif currency_conversion.get("source_type", '').lower() == 'dict': + elif currency_conversion.get("source_type", '').lower() == 'dict': # doesn't work in json as key must be single value return DictBasedCurrencyRates(currency_conversion['currency_rates']) + elif currency_conversion.get("source_type", '').lower() == 'list': # option to write directly the rate in the json file + return DictBasedCurrencyRates.from_list(currency_conversion['currency_rates']) else: raise OdsException( f"Unsupported currency_conversion source type : {currency_conversion.get('source_type')}") @@ -253,21 +255,23 @@ def convert_currency(oed_df, oed_type, reporting_currency, currency_rate, oed_sc oed_df.loc[orig_cur_rows, 'RateOfExchange'] *= rate for field, column in field_to_column.items(): field_type = ods_fields[field.lower()].get('Back End DB Field Name', '').lower() - if (field_type in ['tax', 'grosspremium', 'netpremium', 'brokerage', 'extraexpenselimit', 'minded', - 'maxded'] - or field.endswith('tiv')): + + if ( + field_type in ['tax', 'grosspremium', 'netpremium', 'brokerage', 'extraexpenselimit', 'minded', 'maxded'] + or field.lower().endswith('tiv') + or field in ['LayerLimit', 'LayerAttachment']): row_filter = orig_cur_rows elif field_type == 'ded': - column_type_name = field.replace('ded', 'dedtype') + column_type_name = field.replace('Ded', 'DedType') row_filter = orig_cur_rows & (oed_df[field_to_column[column_type_name]] == 0) elif field_type == 'limit': - column_type_name = field.replace('limit', 'limittype') + column_type_name = field.replace('Limit', 'LimitType') row_filter = orig_cur_rows & (oed_df[field_to_column[column_type_name]] == 0) elif field_type in ['payoutstart', 'payoutend', 'payoutlimit']: - column_type_name = 'payouttype' + column_type_name = 'PayoutType' row_filter = orig_cur_rows & (oed_df[field_to_column[column_type_name]] == 0) elif field_type in ['triggerstart', 'triggerend']: - column_type_name = 'triggertype' + column_type_name = 'TriggerType' row_filter = orig_cur_rows & (oed_df[field_to_column[column_type_name]] == 0) else: # not a currency unit column we go to the next one continue diff --git a/ods_tools/oed/oed_schema.py b/ods_tools/oed/oed_schema.py index 14ee6b67..eba6cb87 100644 --- a/ods_tools/oed/oed_schema.py +++ b/ods_tools/oed/oed_schema.py @@ -1,8 +1,32 @@ import json import os from pathlib import Path +import numba as nb +import numpy as np -from .common import OdsException, BLANK_VALUES +from .common import OdsException, BLANK_VALUES, cached_property + +ENV_ODS_SCHEMA_PATH = os.getenv('ODS_SCHEMA_PATH') + + +# function to check if a subperil is part of a peril +@nb.jit(cache=True, nopython=True) +def __single_peril_filtering(peril_id, peril_filter, perils_dict): + if peril_filter: + for peril_filter in peril_filter.split(';'): + for p in perils_dict[peril_filter]: + if p == peril_id: + return True + return False + + +@nb.jit(cache=True, nopython=True) +def jit_peril_filtering(peril_ids, peril_filters, perils_dict): + result = np.empty_like(peril_ids, dtype=np.bool_) + for i in range(peril_ids.shape[0]): + result[i] = __single_peril_filtering(peril_ids[i], peril_filters[i], perils_dict) + + return result class OedSchema: @@ -14,8 +38,8 @@ class OedSchema: json_path (Path or str): path to the json file from where the schema was loaded """ DEFAULT_ODS_SCHEMA_FILE = 'OpenExposureData_Spec.json' - DEFAULT_ODS_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - 'data', DEFAULT_ODS_SCHEMA_FILE) + DEFAULT_ODS_SCHEMA_PATH = (ENV_ODS_SCHEMA_PATH if ENV_ODS_SCHEMA_PATH + else os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', DEFAULT_ODS_SCHEMA_FILE)) def __init__(self, schema, json_path): """ @@ -75,6 +99,28 @@ def from_json(cls, oed_json): return cls(schema, oed_json) + @cached_property + def nb_perils_dict(self): + nb_perils_dict = nb.typed.Dict.empty( + key_type=nb.types.UnicodeCharSeq(3), + value_type=nb.types.UnicodeCharSeq(3)[:], + ) + for peril_group, perils in self.schema['perils']['covered'].items(): + nb_perils_dict[peril_group] = np.array(perils, dtype='U3') + + return nb_perils_dict + + def peril_filtering(self, peril_ids, peril_filters): + """ + check if peril_ids are part of the peril groups in peril_filters, both array need to match size + Args: + peril_ids (pd.Series): peril that are checked + peril_filters (pd.Series): peril groups to check against + :return: + np.array of True and False + """ + return jit_peril_filtering(peril_ids.to_numpy().astype('str'), peril_filters.to_numpy().astype('str'), self.nb_perils_dict) + @staticmethod def to_universal_field_name(column: str): """ diff --git a/ods_tools/oed/source.py b/ods_tools/oed/source.py index a91ad2b2..3756bf0b 100644 --- a/ods_tools/oed/source.py +++ b/ods_tools/oed/source.py @@ -7,57 +7,12 @@ from chardet.universaldetector import UniversalDetector from .common import (OED_TYPE_TO_NAME, OdsException, PANDAS_COMPRESSION_MAP, PANDAS_DEFAULT_NULL_VALUES, is_relative, BLANK_VALUES, fill_empty, - UnknownColumnSaveOption) + UnknownColumnSaveOption, cached_property) from .forex import convert_currency from .oed_schema import OedSchema logger = logging.getLogger(__file__) -try: - from functools import cached_property -except ImportError: # support for python < 3.8 - _missing = object() - - class cached_property(object): - """A decorator that converts a function into a lazy property. The - function wrapped is called the first time to retrieve the result - and then that calculated result is used the next time you access - the value:: - - class Foo(object): - - @cached_property - def foo(self): - # calculate something important here - return 42 - - The class has to have a `__dict__` in order for this property to - work. - """ - - # implementation detail: this property is implemented as non-data - # descriptor. non-data descriptors are only invoked if there is - # no entry with the same name in the instance's __dict__. - # this allows us to completely get rid of the access function call - # overhead. If one choses to invoke __get__ by hand the property - # will still work as expected because the lookup logic is replicated - # in __get__ for manual invocation. - - def __init__(self, func, name=None, doc=None): - self.__name__ = name or func.__name__ - self.__module__ = func.__module__ - self.__doc__ = doc or func.__doc__ - self.func = func - - def __get__(self, obj, type=None): - if obj is None: - return self - value = obj.__dict__.get(self.__name__, _missing) - if value is _missing: - value = self.func(obj) - obj.__dict__[self.__name__] = value - return value - def detect_encoding(fileobj): """ diff --git a/requirements.in b/requirements.in index 4b27ec87..b5854d24 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,5 @@ pandas +numba chardet jsonschema jsonref diff --git a/setup.py b/setup.py index 291509d9..b4923815 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) -OED_VERSION = '3.0.4' +OED_VERSION = '3.1.0' # ORD_VERSION = diff --git a/tests/test_ods_package.py b/tests/test_ods_package.py index 524a2e19..3ce1d5d9 100644 --- a/tests/test_ods_package.py +++ b/tests/test_ods_package.py @@ -588,3 +588,241 @@ def test_empty_dataframe_logged(self): oed.check() assert 'location ' in self._caplog.text assert 'is empty' in self._caplog.text + + def test_peril_filtering(self): + """check that oed_schema.peril_filtering works correctly""" + loc_df = pd.DataFrame({ + 'PortNumber': [1, 1, 1, 1], + 'AccNumber': [1, 1, 1, 1], + 'LocNumber': [1, 2, 3, 4], + 'CountryCode': ['UK', 'UK', 'UK', 'UK',], + 'LocPerilsCovered': ['WW2', 'WTC;WSS', 'QQ1;WW2', 'WTC'], + 'BuildingTIV': ['1', '1', '1', '1'], + 'ContentsTIV': ['1', '1', '1', '1'], + 'LocCurrency': ['1', '1', '1', '1'], + }) + oed = OedExposure(**{'location': loc_df, 'use_field': True}) + peril_in = pd.Series(['WTC'] * 4) + peril_out = pd.Series(['XLT'] * 4) + assert oed.oed_schema.peril_filtering(peril_in, oed.location.dataframe['LocPerilsCovered']).all() + assert not oed.oed_schema.peril_filtering(peril_out, oed.location.dataframe['LocPerilsCovered']).any() + + def test_to_version_with_invalid_format(self): + oed_exposure = OedExposure( + location=base_url + "/SourceLocOEDPiWind.csv", + account=base_url + "/SourceAccOEDPiWind.csv", + ri_info=base_url + "/SourceReinsInfoOEDPiWind.csv", + ri_scope=base_url + "/SourceReinsScopeOEDPiWind.csv", + use_field=True, + ) + + if "versioning" in oed_exposure.oed_schema.schema: + with pytest.raises(ValueError, match="Invalid version: 3.x"): + oed_exposure.to_version("3.x") + else: + assert True + + def test_versioning_fallback(self): + oed_exposure = OedExposure( + location=base_url + "/SourceLocOEDPiWind.csv", + account=base_url + "/SourceAccOEDPiWind.csv", + ri_info=base_url + "/SourceReinsInfoOEDPiWind.csv", + ri_scope=base_url + "/SourceReinsScopeOEDPiWind.csv", + use_field=True, + ) + + if "versioning" not in oed_exposure.oed_schema.schema: + oed_exposure.oed_schema.schema["versioning"] = {} + + oed_exposure.oed_schema.schema["versioning"] = { + "1.9": [ + { + "Category": "Occupancy", + "New code": 9999, + "Fallback": 9998 + } + ] + } + + # Modify the first line of exposure.location.dataframe + oed_exposure.location.dataframe.loc[0, "OccupancyCode"] = 9999 + + # Convert + oed_exposure.to_version("1.9") + + # # Assert the OccupancyCode is as expected + assert oed_exposure.location.dataframe.loc[0, "OccupancyCode"] == 9998 + + def test_versioning_fallback_not_exact(self): + oed_exposure = OedExposure( + location=base_url + "/SourceLocOEDPiWind.csv", + account=base_url + "/SourceAccOEDPiWind.csv", + ri_info=base_url + "/SourceReinsInfoOEDPiWind.csv", + ri_scope=base_url + "/SourceReinsScopeOEDPiWind.csv", + use_field=True, + ) + + if "versioning" not in oed_exposure.oed_schema.schema: + oed_exposure.oed_schema.schema["versioning"] = {} + + oed_exposure.oed_schema.schema["versioning"] = { + "1.9": [ + { + "Category": "Occupancy", + "New code": 9999, + "Fallback": 9998 + } + ], + "1.5": [ + { + "Category": "Occupancy", + "New code": 9998, + "Fallback": 9997 + } + ], + "1.1": [ + { + "Category": "Occupancy", + "New code": 9997, + "Fallback": 9996 + } + ] + } + + # Modify the first line of exposure.location.dataframe + oed_exposure.location.dataframe.loc[0, "OccupancyCode"] = 9999 + + # Convert + oed_exposure.to_version("1.3") + + # # Assert the OccupancyCode is as expected + assert oed_exposure.location.dataframe.loc[0, "OccupancyCode"] == 9997 + + def test_versioning_higher(self): + oed_exposure = OedExposure( + location=base_url + "/SourceLocOEDPiWind.csv", + account=base_url + "/SourceAccOEDPiWind.csv", + ri_info=base_url + "/SourceReinsInfoOEDPiWind.csv", + ri_scope=base_url + "/SourceReinsScopeOEDPiWind.csv", + use_field=True, + ) + + if "versioning" not in oed_exposure.oed_schema.schema: + oed_exposure.oed_schema.schema["versioning"] = {} + + oed_exposure.oed_schema.schema["versioning"] = { + "1.9": [ + { + "Category": "Occupancy", + "New code": 9999, + "Fallback": 9998 + } + ] + } + + # Modify the first line of exposure.location.dataframe + oed_exposure.location.dataframe.loc[0, "OccupancyCode"] = 9999 + + # Convert + oed_exposure.to_version("1.10") + + # # Assert the OccupancyCode is as expected + assert oed_exposure.location.dataframe.loc[0, "OccupancyCode"] == 9999 + + def test_versioning_lower_than_supported(self): + oed_exposure = OedExposure( + location=base_url + "/SourceLocOEDPiWind.csv", + account=base_url + "/SourceAccOEDPiWind.csv", + ri_info=base_url + "/SourceReinsInfoOEDPiWind.csv", + ri_scope=base_url + "/SourceReinsScopeOEDPiWind.csv", + use_field=True, + ) + + if "versioning" not in oed_exposure.oed_schema.schema: + oed_exposure.oed_schema.schema["versioning"] = {} + + oed_exposure.oed_schema.schema["versioning"] = { + "1.9": [ + { + "Category": "Occupancy", + "New code": 9999, + "Fallback": 9998 + } + ], + "1.5": [ + { + "Category": "Occupancy", + "New code": 9998, + "Fallback": 9997 + } + ], + "1.1": [ + { + "Category": "Occupancy", + "New code": 9997, + "Fallback": 9996 + } + ] + } + + # Modify the first line of exposure.location.dataframe + oed_exposure.location.dataframe.loc[0, "OccupancyCode"] = 9999 + + # Convert + oed_exposure.to_version("0.8") + + # # Assert the OccupancyCode is as expected + assert oed_exposure.location.dataframe.loc[0, "OccupancyCode"] == 9996 + + def test_versioning_wrong_order(self): + oed_exposure = OedExposure( + location=base_url + "/SourceLocOEDPiWind.csv", + account=base_url + "/SourceAccOEDPiWind.csv", + ri_info=base_url + "/SourceReinsInfoOEDPiWind.csv", + ri_scope=base_url + "/SourceReinsScopeOEDPiWind.csv", + use_field=True, + ) + + if "versioning" not in oed_exposure.oed_schema.schema: + oed_exposure.oed_schema.schema["versioning"] = {} + + oed_exposure.oed_schema.schema["versioning"] = { + "1.9": [ + { + "Category": "Occupancy", + "New code": 9997, + "Fallback": 9995 + } + ], + "1.12": [ + { + "Category": "Occupancy", + "New code": 9998, + "Fallback": 9997 + } + ], + "1.1": [ + { + "Category": "Occupancy", + "New code": 9995, + "Fallback": 9993 + } + ], + "2.1": [ + { + "Category": "Occupancy", + "New code": 9990, + "Fallback": 9998 + } + ] + + } + + # Modify the first line of exposure.location.dataframe + oed_exposure.location.dataframe.loc[0, "OccupancyCode"] = 9990 + + # Convert + oed_exposure.to_version("1.8") + + # # Assert the OccupancyCode is as expected + assert oed_exposure.location.dataframe.loc[0, "OccupancyCode"] == 9995