From 43b361146a060b2a88063522c9c91c251b2c11a4 Mon Sep 17 00:00:00 2001 From: Merel Theisen <49397448+merelcht@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:52:33 +0100 Subject: [PATCH] Clarify in docs how runtime parameter resolution works (#4096) * Add clarifications in docs explaining how runtime parameter resolution works. * Add example of how runtime parameters will be merged --------- Signed-off-by: Merel Theisen --- RELEASE.md | 1 + .../configuration/advanced_configuration.md | 3 +- docs/source/configuration/parameters.md | 42 +++++++++- tests/config/test_omegaconf_config.py | 83 +++++++++++++++++++ 4 files changed, 127 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 25802d76f8..d9c788e957 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,6 +10,7 @@ ## Breaking changes to the API ## Documentation changes +* Add clarifications in docs explaining how runtime parameter resolution works. ## Community contributions diff --git a/docs/source/configuration/advanced_configuration.md b/docs/source/configuration/advanced_configuration.md index 4d547e619a..914f5f44ae 100644 --- a/docs/source/configuration/advanced_configuration.md +++ b/docs/source/configuration/advanced_configuration.md @@ -322,7 +322,8 @@ Note that you can only use the resolver in `credentials.yml` and not in catalog ``` ### How to change the merge strategy used by `OmegaConfigLoader` -By default, `OmegaConfigLoader` merges configuration [in different environments](configuration_basics.md#configuration-environments) in a destructive way. This means that whatever configuration resides in your overriding environment (`local` by default) takes precedence when the same top-level key is present in the base and overriding environment. Any configuration for that key **besides that given in the overriding environment** is discarded. +By default, `OmegaConfigLoader` merges configuration [in different environments](configuration_basics.md#configuration-environments) as well as runtime parameters in a destructive way. This means that whatever configuration resides in your overriding environment (`local` by default) takes precedence when the same top-level key is present in the base and overriding environment. Any configuration for that key **besides that given in the overriding environment** is discarded. +The same behaviour applies to runtime parameters overriding any configuration in the `base` environment. You can change the merge strategy for each configuration type in your project's `src//settings.py`. The accepted merging strategies are `soft` and `destructive`. ```python diff --git a/docs/source/configuration/parameters.md b/docs/source/configuration/parameters.md index 80abb7eece..16d1514fec 100644 --- a/docs/source/configuration/parameters.md +++ b/docs/source/configuration/parameters.md @@ -117,7 +117,47 @@ Each key-value pair is split on the first equals sign. The following example is ```bash kedro run --params=param_key1=value1,param_key2=2.0 ``` -Values provided in the CLI take precedence and overwrite parameters specified in configuration files. +Values provided in the CLI take precedence and overwrite parameters specified in configuration files. By default, runtime parameters get merged destructively, meaning that any configuration for that key **besides that given in the runtime parameters** is discarded. +[This section describes how to change the merging strategy](advanced_configuration.md#how-to-change-the-merge-strategy-used-by-omegaconfigloader). + +For example, if you have the following parameters in your `base` and `local` environments: + +```yaml +# base/parameters.yml +model_options: + model_params: + learning_date: "2023-11-01" + training_date: "2023-11-01" + data_ratio: 14 + +data_options: + step_size: 123123 +``` + +```yaml +# local/parameters.yml +features: + rate: 123 +``` + +And you provide the following parameter at runtime: + +```bash +kedro run --params="model_options.model_params.training_date=2011-11-11" +``` + +The final merged result will be: +```yaml +model_options: + model_params: + training_date: "2011-11-11" + +data_options: + step_size: 123123 + +features: + rate: 123 +``` * Parameter keys are _always_ treated as strings. * Parameter values are converted to a float or an integer number if the corresponding conversion succeeds; otherwise, they are also treated as string. diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index c2e509d9bf..1eb4d70cb8 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1068,6 +1068,89 @@ def test_runtime_params_resolution(self, tmp_path): # runtime params are resolved correctly in catalog assert conf["catalog"]["companies"]["type"] == runtime_params["dataset"]["type"] + def test_runtime_params_resolution_with_soft_merge_base_env(self, tmp_path): + """Test that runtime_params get softly merged with the base environment when soft merge is set + for parameter merge""" + base_params = tmp_path / _BASE_ENV / "parameters.yml" + prod_params = tmp_path / "prod" / "parameters.yml" + runtime_params = { + "aaa": { + "bbb": { + "abb": "2011-11-11", + } + } + } + param_config = { + "aaa": { + "bbb": { + "aba": "2023-11-01", + "abb": "2023-11-01", + "abc": 14, + } + }, + "xyz": {"asdf": 123123}, + } + prod_param_config = {"def": {"gg": 123}} + _write_yaml(base_params, param_config) + _write_yaml(prod_params, prod_param_config) + conf = OmegaConfigLoader( + tmp_path, + base_env=_BASE_ENV, + default_run_env="prod", + runtime_params=runtime_params, + merge_strategy={"parameters": "soft"}, + ) + + expected_parameters = { + "aaa": {"bbb": {"aba": "2023-11-01", "abb": "2011-11-11", "abc": 14}}, + "xyz": {"asdf": 123123}, + "def": {"gg": 123}, + } + + # runtime parameters are resolved correctly across parameter files from different environments + assert conf["parameters"] == expected_parameters + + def test_runtime_params_resolution_default_run_env(self, tmp_path): + """Test that runtime_params overwrite merge with the default run environment""" + base_params = tmp_path / _BASE_ENV / "parameters.yml" + prod_params = tmp_path / "prod" / "parameters.yml" + runtime_params = {"data_shift": 3} + param_config = { + "model_options": { + "test_size": 0.2, + "random_state": 3, + "features": ["engines"], + } + } + prod_param_config = { + "model_options": { + "test_size": 0.2, + "random_state": 3, + "features": ["engines"], + }, + "data_shift": 1, + } + _write_yaml(base_params, param_config) + _write_yaml(prod_params, prod_param_config) + conf = OmegaConfigLoader( + tmp_path, + base_env=_BASE_ENV, + default_run_env="prod", + runtime_params=runtime_params, + ) + + expected_parameters = { + "model_options": { + "test_size": 0.2, + "random_state": 3, + "features": ["engines"], + }, + "data_shift": 3, + } + + # runtime parameters are resolved correctly across parameter files from different environments + assert conf["parameters"] == expected_parameters + def test_runtime_params_missing_default(self, tmp_path): base_params = tmp_path / _BASE_ENV / "parameters.yml" runtime_params = {