diff --git a/.github/workflows/pr-ci-pipbuild-environment.yml b/.github/workflows/pr-ci-pipbuild-environment.yml new file mode 100644 index 00000000..c18576ea --- /dev/null +++ b/.github/workflows/pr-ci-pipbuild-environment.yml @@ -0,0 +1,8 @@ +name: pipbuild +channels: + - conda-forge +dependencies: + - python=3.11 + - pip + - pip: + - build diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 747e6e6c..c989b30c 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -109,11 +109,7 @@ jobs: - uses: mamba-org/setup-micromamba@v1 with: micromamba-version: latest - environment-name: pipbuild - create-args: >- - python=3.11 - pip - build + environment-file: .github/workflows/pr-ci-pipbuild-environment.yml post-cleanup: all cache-environment: true - name: Build package diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 608e1666..67779e53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # https://beta.ruff.rs/docs/usage/#github-action - rev: v0.5.0 + rev: v0.6.3 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/AUTHORS b/AUTHORS index 70c3870b..7b1de98f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,3 +13,4 @@ Francesco Lombardi, Politecnico di Milano Adriaan Hilbers, Imperial College London Jann Launer, TU Delft Ivan Ruiz Manuel, TU Delft +Stefan Strömer, AIT Austrian Institute of Technology GmbH and TU Delft diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e42bc22..190c5615 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,30 @@ -## 0.7.0.dev4 (dev) +## 0.7.0.dev5 (Unreleased) ### User-facing changes |new| Math has been removed from `model.math`, and can now be accessed via `model.math.data`. +## 0.7.0.dev4 (2024-09-10) + +### User-facing changes + +|fixed| Decision variable domain in math docs to use $\in$ instead of $\forall$ (#652). + +|fixed| Clarity of `flow_cap_min` description in documentation (#653). + +|changed| API/schema documentation is de-ranked in documentation search bar results (#670). + +|new| Math component cross-references in both directions ("uses" and "used in") in Markdown math documentation (#643). + +|fixed| Duplicated links in math documentation (#651). + +|changed| `node_groups` and `tech_groups` changed to a general top-level `templates` key, +accessed via the `template` key (replacing `inherit`) in `nodes` and `techs` (#600). + +|fixed| Contribution of `cost_om_annual_investment_fraction` to total investment costs, to not apply to depreciated costs (#645). + +|fixed| Math for multi-carrier variable export costs (#663). + |new| Piecewise constraints added to the YAML math with its own unique syntax (#107). These constraints will be added to the optimisation problem using Special Ordered Sets of Type 2 (SOS2) variables. diff --git a/docs/advanced/mode.md b/docs/advanced/mode.md index 55cc3685..8a7701f4 100644 --- a/docs/advanced/mode.md +++ b/docs/advanced/mode.md @@ -98,9 +98,9 @@ Technologies at locations with higher scores will be penalised in the objective In the [national scale example model](../examples/national_scale/index.md), this would look something like: ```yaml -tech_groups: +templates: add_spores_score: - inherit: cost_dim_setter + template: cost_dim_setter cost_flow_cap: data: [null, null] index: ["monetary", "spores_score"] @@ -112,16 +112,16 @@ tech_groups: techs: ccgt: - inherit: add_spores_score + template: add_spores_score cost_flow_cap.data: [750, 0] csp: - inherit: add_spores_score + template: add_spores_score cost_flow_cap.data: [1000, 0] battery: - inherit: add_spores_score + template: add_spores_score cost_flow_cap.data: [null, 0] region1_to_region2: - inherit: add_spores_score + template: add_spores_score cost_flow_cap.data: [10000, 0] ``` diff --git a/docs/advanced/time.md b/docs/advanced/time.md index 65e8fe79..52cc9cb3 100644 --- a/docs/advanced/time.md +++ b/docs/advanced/time.md @@ -31,6 +31,58 @@ This particular math - detailed [here][inter-cluster-storage-math] - can be enab We no longer provide the functionality to infer representative days from your timeseries. Instead, we recommend you use other timeseries processing tools applied to your input CSV data or your built model dataset (`model.inputs`). +If you want cluster the timeseries data yourself, we recommend these tools: + +* [tsam](https://github.com/FZJ-IEK3-VSA/tsam): designed specifically for large-scale energy system models. +* [scikit-learn](https://scikit-learn.org/stable/): a general machine learning library that has a clustering module. +We were previously using this in our internal clustering. +* [tslearn](https://tslearn.readthedocs.io/en/stable/index.html): a timeseries-focussed machine learning library. + +??? example "Using the `tsam` library to cluster your timeseries" + + In this example, we will find 12 representative days of a clustered timeseries and save those to file. + For more configuration options, see the [`tsam` documentation](https://tsam.readthedocs.io/en/). + + ```python + import tsam.timeseriesaggregation as tsam + import calliope + + # Load data at full time resolution + model = calliope.Model(...) + # Get all timeseries data from model, with timesteps on the rows and all other dimensions on the columns + raw_data = ( + model.inputs[[ + k for k, v in model.inputs.data_vars.items() + if "timesteps" in v.dims and len(v.dims) > 1 + ]] + .to_dataframe() + .stack() + .unstack("timesteps") + .T + ) + aggregation = tsam.TimeSeriesAggregation( + raw_data, noTypicalPeriods=12, hoursPerPeriod=24, clusterMethod="hierarchical" + ) + typPeriods = aggregation.createTypicalPeriods() + matched_indices = aggregation.indexMatching() + representative_dates = ( + raw_data + .resample("1D") + .first() + .iloc[aggregation.clusterCenterIndices] + .index + ) + cluster_days = ( + matched_indices + .resample("1D") + .first() + .PeriodNum + .apply(lambda x: representative_dates[x]) + ) + cluster_days.to_csv("/absolute_path/to/clusters.csv") + + model_clustered = calliope.Model(..., time_cluster="/absolute_path/to/clusters.csv") + ``` !!! note diff --git a/docs/creating/config.md b/docs/creating/config.md index ab04488b..ed2a90ac 100644 --- a/docs/creating/config.md +++ b/docs/creating/config.md @@ -37,6 +37,10 @@ None of the configuration options are _required_ as there is a default value for To test your model pipeline, `config.init.time_subset` is a good way to limit your model size by slicing the time dimension to a smaller range. +!!! note + Various capabilities are available to adjust the temporal resolution of a model on-the-fly, both by resampling or using externally-provided clustering. + See our [time adjustment page](../advanced/time.md) for more details. + ## Deep-dive into some key configuration options ### `config.build.backend` diff --git a/docs/creating/data_sources.md b/docs/creating/data_sources.md index b6926268..f8735ed7 100644 --- a/docs/creating/data_sources.md +++ b/docs/creating/data_sources.md @@ -235,7 +235,7 @@ In this section we will show some examples of loading data and provide the equiv === "YAML" ```yaml - tech_groups: # (1)! + templates: # (1)! cost_setter: cost_interest_rate: data: 0.1 @@ -260,7 +260,7 @@ In this section we will show some examples of loading data and provide the equiv techs: tech1: - inherit: cost_setter + template: cost_setter cost_flow_cap.data: 100 cost_area_use.data: 50 cost_flow_out.data: 0.2 @@ -269,7 +269,7 @@ In this section we will show some examples of loading data and provide the equiv cost_storage_cap.data: 150 ``` - 1. To limit repetition, we have defined [technology groups](groups.md) for our costs. + 1. To limit repetition, we have defined [templates](templates.md) for our costs. !!! info "See also" Our [data source loading tutorial][loading-tabular-data] has more examples of loading tabular data into your model. diff --git a/docs/creating/index.md b/docs/creating/index.md index 9c8bc052..c8af0587 100644 --- a/docs/creating/index.md +++ b/docs/creating/index.md @@ -35,7 +35,7 @@ We distinguish between: - the model **definition** (your representation of a physical system in YAML). Model configuration is everything under the top-level YAML key [`config`](config.md). -Model definition is everything else, under the top-level YAML keys [`parameters`](parameters.md), [`techs`](techs.md), [`nodes`](nodes.md), [`tech_groups`](groups.md), [`node_groups`](groups.md), and [`data_sources`](data_sources.md). +Model definition is everything else, under the top-level YAML keys [`parameters`](parameters.md), [`techs`](techs.md), [`nodes`](nodes.md), [`templates`](templates.md), and [`data_sources`](data_sources.md). It is possible to define alternatives to the model configuration/definition that you can refer to when you initialise your model. These are defined under the top-level YAML keys [`scenarios` and `overrides`](scenarios.md). @@ -84,5 +84,5 @@ The rest of this section discusses everything you need to know to set up a model - An overview of [YAML as it is used in Calliope](yaml.md) - though this comes first here, you can also safely skip it and refer back to it as a reference as questions arise when you go through the model configuration and definition examples. - More details on the [model configuration](config.md). - The key parts of the model definition, first, the [technologies](techs.md), then, the [nodes](nodes.md), the locations in space where technologies can be placed. -- How to use [technology and node inheritance](groups.md) to reduce repetition in the model definition. +- How to use [technology and node templates](templates.md) to reduce repetition in the model definition. - Other important features to be aware of when defining your model: defining [indexed parameters](parameters.md), i.e. parameter which are not indexed over technologies and nodes, [loading tabular data](data_sources.md), and defining [scenarios and overrides](scenarios.md). diff --git a/docs/creating/techs.md b/docs/creating/techs.md index b4c783a3..60d0479d 100644 --- a/docs/creating/techs.md +++ b/docs/creating/techs.md @@ -12,12 +12,13 @@ This establishes the basic characteristics in the optimisation model (decision v * `transmission`: Transmits a carrier from one node to another. * `conversion`: Converts from one carrier to another. -??? info "Sharing configuration through inheritance" - To share definitions between technologies and/or nodes, you can use configuration inheritance (the `inherit` key). - This allows a technology/node to inherit definitions from [`tech_group`/`node_group` definitions](groups.md). - Note that `inherit` is different to setting a `base_tech`. - Setting a base_tech does not entail any configuration options being inherited. - It is only used when building the optimisation problem (i.e., in the `math`). +??? info "Sharing configuration with templates" + + To share definitions between technologies and/or nodes, you can use configuration templates (the `template` key). + This allows a technology/node to inherit definitions from [`template` definitions](templates.md). + Note that `template` is different to setting a `base_tech`. + Setting a base_tech does not entail any configuration options being inherited; + `base_tech` is only used when building the optimisation problem (i.e., in the `math`). The following example shows the definition of a `ccgt` technology, i.e. a combined cycle gas turbine that delivers electricity: @@ -193,4 +194,4 @@ In an [override](scenarios.md) you may want to remove a technology entirely from The easiest way to do this is to set `active: false`. The resulting input dataset won't feature that technology in any way. You can even do this to deactivate technologies at specific [nodes](nodes.md) and to deactivate nodes entirely. -Conversely, setting `active: true` in an override will lead to the technology reappearing. \ No newline at end of file +Conversely, setting `active: true` in an override will lead to the technology reappearing. diff --git a/docs/creating/groups.md b/docs/creating/templates.md similarity index 70% rename from docs/creating/groups.md rename to docs/creating/templates.md index d076c9b2..d1d970dd 100644 --- a/docs/creating/groups.md +++ b/docs/creating/templates.md @@ -1,13 +1,13 @@ -# Inheriting from technology node groups: `tech_groups`, `node_groups` +# Inheriting from templates: `templates` For larger models, duplicate entries can start to crop up and become cumbersome. -To streamline data entry, technologies and nodes can inherit common data from a `tech_group` or `node_group`, respectively. +To streamline data entry, technologies and nodes can inherit common data from a `template`. For example, if we want to set interest rate to `0.1` across all our technologies, we could define: ```yaml -tech_groups: +templates: interest_rate_setter: cost_interest_rate: data: 0.1 @@ -15,29 +15,29 @@ tech_groups: dims: costs techs: ccgt: - inherit: interest_rate_setter + template: interest_rate_setter ... ac_transmission: - inherit: interest_rate_setter + template: interest_rate_setter ... ``` Similarly, if we want to allow the same technologies at all our nodes: ```yaml -node_groups: +templates: standard_tech_list: techs: {ccgt, battery, demand_power} # (1)! nodes: region1: - inherit: standard_tech_list + template: standard_tech_list ... region2: - inherit: standard_tech_list + template: standard_tech_list ... ... region100: - inherit: standard_tech_list + template: standard_tech_list ``` 1. this YAML syntax is shortform for: @@ -48,19 +48,19 @@ nodes: demand_power: ``` -Inheritance chains can also be set up. -That is, groups can inherit from groups. +Inheritance chains can also be created. +That is, templates can inherit from other templates. E.g.: ```yaml -tech_groups: +templates: interest_rate_setter: cost_interest_rate: data: 0.1 index: monetary dims: costs investment_cost_setter: - inherit: interest_rate_setter + template: interest_rate_setter cost_flow_cap: data: 100 index: monetary @@ -71,25 +71,25 @@ tech_groups: dims: costs techs: ccgt: - inherit: investment_cost_setter + template: investment_cost_setter ... ac_transmission: - inherit: interest_rate_setter + template: interest_rate_setter ... ``` -Finally, inherited properties can always be overridden by the inheriting component. +Finally, template properties can always be overridden by the inheriting component. This can be useful to streamline setting costs, e.g.: ```yaml -tech_groups: +templates: interest_rate_setter: cost_interest_rate: data: 0.1 index: monetary dims: costs investment_cost_setter: - inherit: interest_rate_setter + template: interest_rate_setter cost_interest_rate.data: 0.2 # this will replace `0.1` in the `interest_rate_setter`. cost_flow_cap: data: null @@ -101,7 +101,7 @@ tech_groups: dims: costs techs: ccgt: - inherit: investment_cost_setter + template: investment_cost_setter cost_flow_cap.data: 100 # this will replace `null` in the `investment_cost_setter`. ... ``` diff --git a/docs/examples/index.md b/docs/examples/index.md index 90f296a1..85ee4cca 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -16,6 +16,6 @@ The ["urban scale" example](urban_scale/index.md) builds a model for part of a d * Use of conversion technologies with singular and multiple output carriers. * Revenue generation, by carrier export. -* Inheriting from technology groups +* Inheriting from templates The ["MILP" example](milp/index.md) extends the urban scale example, exhibiting binary and integer decision variable functionality (extended an LP model to a MILP model). diff --git a/docs/examples/loading_tabular_data.py b/docs/examples/loading_tabular_data.py index 8dcd3327..edaaf819 100644 --- a/docs/examples/loading_tabular_data.py +++ b/docs/examples/loading_tabular_data.py @@ -22,9 +22,10 @@ # %% from pathlib import Path -import calliope import pandas as pd +import calliope + calliope.set_log_verbosity("INFO", include_solver_output=False) # %% [markdown] diff --git a/docs/examples/national_scale/index.md b/docs/examples/national_scale/index.md index 8a7f599a..25a1b46d 100644 --- a/docs/examples/national_scale/index.md +++ b/docs/examples/national_scale/index.md @@ -110,16 +110,16 @@ The costs are more numerous as well, and include monetary costs for all relevant * carrier conversion capacity * variable operational and maintenance costs -### Interlude: inheriting from technology groups +### Interlude: inheriting from templates You will notice that the above technologies _inherit_ `cost_dim_setter`. -Inheritance allows us to avoid excessive repetition in our model definition. +Templates allow us to avoid excessive repetition in our model definition. In this case, `cost_dim_setter` defines the dimension and index of costs, allowing us to keep our definition of technology costs to only defining `data`. By defining `data`, the technologies override the `null` setting applied by `cost_dim_setter`. We also use it to set the `interest_rate` for all technologies, which will be used to annualise any investment costs each technology defines. -Technologies can inherit from anything defined in `tech_groups`, while nodes can inherit from anything in `node_groups`. -items in `[tech/node]_groups` can also inherit from each other, so you can create inheritance chains. +Technologies and nodes can inherit from anything defined in `templates`. +items in `templates` can also inherit from each other, so you can create inheritance chains. `cost_dim_setter` looks like this: @@ -185,7 +185,7 @@ Transmission technologies look different to other technologies, as they link the As the name suggests, it applies no cost or efficiency losses to this transmission. We can see that those technologies which rely on `free_transmission` inherit a lot of this information from elsewhere in the model definition. -`free_transmission` is defined in `tech_groups`, which makes it inheritable. +`free_transmission` is defined in `templates`, which makes it inheritable. ```yaml --8<-- "src/calliope/example_models/national_scale/model_config/techs.yaml:free-transmission" @@ -234,11 +234,11 @@ The remaining nodes look similar: ``` `region2` is very similar to `region1`, except that it does not include the `ccgt` technology. -The three `region1-` locations are defined together using the node group `csp_regions`, except for their geospatial coordinates. +The three `region1-` locations are defined together using the template `csp_regions`, except for their geospatial coordinates. They allow only the `csp` technology, this allows us to model three possible sites for CSP plants. ```yaml ---8<-- "src/calliope/example_models/national_scale/model_config/locations.yaml:node-groups" +--8<-- "src/calliope/example_models/national_scale/model_config/locations.yaml:templates" ``` --- diff --git a/docs/examples/national_scale/notebook.py b/docs/examples/national_scale/notebook.py index e6de74a4..70e6d114 100644 --- a/docs/examples/national_scale/notebook.py +++ b/docs/examples/national_scale/notebook.py @@ -18,10 +18,11 @@ # This notebook will show you how to load, build, solve, and examine the results of the national scale example model. # %% -import calliope import pandas as pd import plotly.express as px +import calliope + # We increase logging verbosity calliope.set_log_verbosity("INFO", include_solver_output=False) diff --git a/docs/examples/piecewise_constraints.py b/docs/examples/piecewise_constraints.py index 67dc8c6a..064be53d 100644 --- a/docs/examples/piecewise_constraints.py +++ b/docs/examples/piecewise_constraints.py @@ -21,10 +21,11 @@ # %% -import calliope import numpy as np import plotly.express as px +import calliope + calliope.set_log_verbosity("INFO", include_solver_output=False) # %% [markdown] diff --git a/docs/examples/urban_scale/index.md b/docs/examples/urban_scale/index.md index e083bf32..318cc7f2 100644 --- a/docs/examples/urban_scale/index.md +++ b/docs/examples/urban_scale/index.md @@ -37,7 +37,7 @@ You can find out more about this user-defined math [below](#interlude-user-defin ### Bringing the YAML files together Technically, you could define everything about your model in the same file as your configuration. -One file with the top-level keys `config`, `parameters`, `techs`, `nodes`, `tech_groups`, `node_groups`, `scenarios`, `overrides`. +One file with the top-level keys `config`, `parameters`, `techs`, `nodes`, `templates`, `scenarios`, `overrides`. However, this tends to become unwieldy. Instead, various parts of the model are defined in different files and then we `import` them in the YAML file that we are going to load into calliope (`calliope.Model("my_main_model_file.yaml")`). @@ -127,14 +127,14 @@ The definition of this technology in the example model's configuration looks as --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:pv" ``` -### Interlude: inheriting from technology groups +### Interlude: inheriting from templates You will notice that the above technologies _inherit_ `interest_rate_setter`. -Inheritance allows us to avoid excessive repetition in our model definition. +Templates allow us to avoid excessive repetition in our model definition. In this case, `interest_rate_setter` defines an interest rate that will be used to annualise any investment costs the technology defines. -Technologies can inherit from anything defined in `tech_groups`, while nodes can inherit from anything in `node_groups`. -items in `[tech/node]_groups` can also inherit from each other, so you can create inheritance chains. +Technologies / nodes can inherit from anything defined in `templates`. +items in `templates` can also inherit from each other, so you can create inheritance chains. `interest_rate_setter` looks like this: @@ -241,10 +241,10 @@ Gas is made available in each node without consideration of transmission. --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission" ``` -To avoid excessive duplication in model definition, our transmission technologies inherit most of the their parameters from technology _groups_: +To avoid excessive duplication in model definition, our transmission technologies inherit most of the their parameters from _templates_: ```yaml ---8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission-tech-groups" +--8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission-templates" ``` `power_lines` has an efficiency of 0.95, so a loss during transmission of 0.05. @@ -316,4 +316,4 @@ These revenue possibilities are reflected in the technologies' and locations' de --- !!! info "Where to go next" To try loading and solving the model yourself, move on to the accompanying notebook [here][running-the-urban-scale-example-model]. - You can also find a list of all the example models available in Calliope [here][calliope.examples]. \ No newline at end of file + You can also find a list of all the example models available in Calliope [here][calliope.examples]. diff --git a/docs/examples/urban_scale/notebook.py b/docs/examples/urban_scale/notebook.py index 9cc45947..4374fce9 100644 --- a/docs/examples/urban_scale/notebook.py +++ b/docs/examples/urban_scale/notebook.py @@ -18,10 +18,11 @@ # This notebook will show you how to load, build, solve, and examine the results of the urban scale example model. # %% -import calliope import pandas as pd import plotly.express as px +import calliope + # We increase logging verbosity calliope.set_log_verbosity("INFO", include_solver_output=False) diff --git a/docs/hooks/generate_math_docs.py b/docs/hooks/generate_math_docs.py index 52096ec3..257d7e29 100644 --- a/docs/hooks/generate_math_docs.py +++ b/docs/hooks/generate_math_docs.py @@ -8,9 +8,10 @@ import textwrap from pathlib import Path +from mkdocs.structure.files import File + import calliope from calliope.postprocess.math_documentation import MathDocumentation -from mkdocs.structure.files import File logger = logging.getLogger("mkdocs") @@ -123,9 +124,8 @@ def write_file( nav_reference["Pre-defined math"].append(output_file.as_posix()) - md_doc = math_documentation.write(format="md", mkdocs_tabbed=True) - title = math_documentation.name + math_doc = math_documentation.write(format="md", mkdocs_features=True) file_to_download = Path("..") / filename output_full_filepath.write_text( PREPEND_SNIPPET.format( @@ -134,7 +134,7 @@ def write_file( math_type=title.lower(), filepath=file_to_download, ) - + md_doc + + math_doc ) diff --git a/docs/hooks/generate_plots.py b/docs/hooks/generate_plots.py index 9a606adc..256085f9 100644 --- a/docs/hooks/generate_plots.py +++ b/docs/hooks/generate_plots.py @@ -5,12 +5,13 @@ import tempfile from pathlib import Path -import calliope import pandas as pd import plotly.graph_objects as go import xarray as xr from mkdocs.structure.files import File +import calliope + TEMPDIR = tempfile.TemporaryDirectory() diff --git a/docs/hooks/generate_readable_schema.py b/docs/hooks/generate_readable_schema.py index 19ee6d32..296242e2 100644 --- a/docs/hooks/generate_readable_schema.py +++ b/docs/hooks/generate_readable_schema.py @@ -8,12 +8,14 @@ """ import tempfile +import textwrap from pathlib import Path import jsonschema2md -from calliope.util import schema from mkdocs.structure.files import File +from calliope.util import schema + TEMPDIR = tempfile.TemporaryDirectory() SCHEMAS = { @@ -55,7 +57,16 @@ def _schema_to_md(schema: dict, filename: str, config: dict) -> File: assert lines[2] == "## Properties\n\n" del lines[2] - output_full_filepath.write_text("\n".join(lines)) + initial_lines = textwrap.dedent( + """ + --- + search: + boost: 0.25 + --- + + """ + ) + output_full_filepath.write_text(initial_lines.lstrip() + "\n".join(lines)) return File( path=output_file, diff --git a/docs/migrating.md b/docs/migrating.md index 8dec46d2..f033fb55 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -181,10 +181,10 @@ This split means you can change configuration options on-the-fly if you are work `locations` (abbreviated to `locs` in the Calliope data dimensions) has been renamed to `nodes` (no abbreviation). This allows us to not require an abbreviation and is a disambiguation from the [pandas.DataFrame.loc][] and [xarray.DataArray.loc][] methods. -### `parent` → `base_tech` + `inherit` +### `parent` → `base_tech` + `template` Technology inheritance has been unlinked from its abstract "base" technology. -`inherit` allows for inheriting attributes from `tech_groups` while `base_tech` is fixed to be one of [`demand`, `supply`, `conversion`, `transmission`, `storage`]. +`template` allows for inheriting attributes from `templates` while `base_tech` is fixed to be one of [`demand`, `supply`, `conversion`, `transmission`, `storage`]. === "v0.6" @@ -214,7 +214,7 @@ Technology inheritance has been unlinked from its abstract "base" technology. === "v0.7" ```yaml - tech_groups: + templates: common_interest_rate: cost_interest_rate: data: 0.1 @@ -223,10 +223,10 @@ Technology inheritance has been unlinked from its abstract "base" technology. techs: supply_tech: base_tech: supply - inherit: common_interest_rate + template: common_interest_rate conversion_tech: base_tech: conversion - inherit: common_interest_rate + template: common_interest_rate ``` ### `costs.monetary.flow_cap` → `cost_flow_cap` @@ -309,7 +309,7 @@ Instead, links are defined as separate transmission technologies in `techs`, inc ``` !!! note - You can use [`tech_groups`](creating/groups.md) to minimise duplications in the new transmission technology definition. + You can use [`templates`](creating/templates.md) to minimise duplications in the new transmission technology definition. ### Renaming parameters/decision variables without core changes in function @@ -587,7 +587,7 @@ Instead, you should define your coordinates using [`latitude`/`longitude`](#defi Defining duplicate definitions for nodes by chaining their names in the YAML key (`node1,node2,node3: ...`) is no longer possible. We are trying to minimise the custom elements of our YAML files which allows us to leverage YAML schemas to validate user inputs and to keep our YAML readers more maintainable. -You can now use [`node_groups`](#node_groups) to minimise duplicating key-value pairs in your YAML definitions. +You can now use [`templates`](#templates-for-nodes) to minimise duplicating key-value pairs in your YAML definitions. ### `supply_plus` and `conversion_plus` technology base classes @@ -714,18 +714,11 @@ Time masking and clustering capabilities have been severely reduced. Time resampling and clustering are now accessible by top-level configuration keys: e.g., `config.init.time_resample: 2H`, `config.init.time_cluster: cluster_file.csv`. Clustering is simplified to only matching model dates to representative days, with those representative days being in the clustered timeseries. -If you want to masking/cluster data you should now leverage other tools. +If you want to masking/cluster data you should now leverage other tools, some of which you can find referenced on our [time adjustment](advanced/time.md#time-clustering) page. We made this decision due to the complex nature of time clustering. With our former implementation, we were making decisions about the data that the user should have more awareness of and control over. It is also a constantly evolving field, but not the focus of Calliope, so we are liable to fall behind on the best-in-class methods. -If you want cluster the timeseries data yourself, we recommend these tools: - -* [tsam](https://github.com/FZJ-IEK3-VSA/tsam): designed specifically for large-scale energy system models. -* [scikit-learn](https://scikit-learn.org/stable/): a general machine learning library that has a clustering module. -We were previously using this in our internal clustering. -* [tslearn](https://tslearn.readthedocs.io/en/stable/index.html): a timeseries-focussed machine learning library. - ## Additions ### Storage buffers in all technology base classes @@ -745,9 +738,9 @@ This means you could define different output carriers for a `supply` technology, !!! warning Although our math should be set up to handle multiple carriers and different inflow/outflow carriers for non-conversion technologies, we do not have any direct tests to check possible edge cases. -### `node_groups` +### `templates` for nodes -`node_groups` is the equivalent of `tech_groups` for inheritance of attributes in `nodes`. +The new [`templates` key](creating/templates.md) can be applied to `nodes` as well as `techs`. This makes up for the [removal of grouping node names in keys by comma separation](#comma-separated-node-definitions). So, to achieve this result: @@ -787,16 +780,16 @@ We would do: === "v0.7" ```yaml - node_groups: + templates: standard_tech_list: techs: battery: demand_electricity: ccgt: nodes: - region1.inherit: standard_tech_list - region2.inherit: standard_tech_list - region3.inherit: standard_tech_list + region1.template: standard_tech_list + region2.template: standard_tech_list + region3.template: standard_tech_list ``` ### Inflow and outflow efficiencies diff --git a/docs/reference/api/attrdict.md b/docs/reference/api/attrdict.md index fb327d0c..0ce310ee 100644 --- a/docs/reference/api/attrdict.md +++ b/docs/reference/api/attrdict.md @@ -1 +1,6 @@ -::: calliope.attrdict.AttrDict \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.attrdict.AttrDict diff --git a/docs/reference/api/backend_model.md b/docs/reference/api/backend_model.md index e075bd6c..45f4080e 100644 --- a/docs/reference/api/backend_model.md +++ b/docs/reference/api/backend_model.md @@ -1 +1,6 @@ -::: calliope.backend.backend_model.BackendModel \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.backend.backend_model.BackendModel diff --git a/docs/reference/api/example_models.md b/docs/reference/api/example_models.md index 9af2e23d..e98e8ad1 100644 --- a/docs/reference/api/example_models.md +++ b/docs/reference/api/example_models.md @@ -1 +1,6 @@ -::: calliope.examples \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.examples diff --git a/docs/reference/api/exceptions.md b/docs/reference/api/exceptions.md index 8b501da2..7bfc04cf 100644 --- a/docs/reference/api/exceptions.md +++ b/docs/reference/api/exceptions.md @@ -1 +1,6 @@ -::: calliope.exceptions \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.exceptions diff --git a/docs/reference/api/helper_functions.md b/docs/reference/api/helper_functions.md index 91e03e1c..7ecfc5b0 100644 --- a/docs/reference/api/helper_functions.md +++ b/docs/reference/api/helper_functions.md @@ -1 +1,6 @@ -::: calliope.backend.helper_functions \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.backend.helper_functions diff --git a/docs/reference/api/logging.md b/docs/reference/api/logging.md index 0647613b..654de738 100644 --- a/docs/reference/api/logging.md +++ b/docs/reference/api/logging.md @@ -1 +1,6 @@ -::: calliope.util.logging \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.util.logging diff --git a/docs/reference/api/model.md b/docs/reference/api/model.md index 696588f9..a76cdc02 100644 --- a/docs/reference/api/model.md +++ b/docs/reference/api/model.md @@ -1 +1,6 @@ -::: calliope.Model \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.Model diff --git a/docs/reference/api/schema.md b/docs/reference/api/schema.md index a3f46cfc..99ade3e1 100644 --- a/docs/reference/api/schema.md +++ b/docs/reference/api/schema.md @@ -1 +1,6 @@ -::: calliope.util.schema \ No newline at end of file +--- +search: + boost: 0.5 +--- + +::: calliope.util.schema diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 90538b72..4c3a4ad9 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1,3 +1,8 @@ +--- +search: + boost: 0.5 +--- + # CLI Reference This page provides documentation for the `calliope` command line tool. diff --git a/docs/user_defined_math/customise.md b/docs/user_defined_math/customise.md index b5c44c2a..19131ba8 100644 --- a/docs/user_defined_math/customise.md +++ b/docs/user_defined_math/customise.md @@ -121,5 +121,7 @@ We recommend you only use HTML as the equations can become too long for a PDF pa !!! note - You can add the tabs to flip between rich-text math and the input YAML snippet in your math documentation by using the `mkdocs_tabbed` argument in `math_documentation.write`. + You can add interactive elements to your documentation, if you are planning to host them online using MKDocs. + This includes tabs to flip between rich-text math and the input YAML snippet, and dropdown lists for math component cross-references. + Just set the `mkdocs_features` argument to `True` in `math_documentation.write`. We use this functionality in our [pre-defined math](../pre_defined_math/index.md). diff --git a/docs/user_defined_math/syntax.md b/docs/user_defined_math/syntax.md index be3a4c95..b2266d23 100644 --- a/docs/user_defined_math/syntax.md +++ b/docs/user_defined_math/syntax.md @@ -52,13 +52,13 @@ Configuration options are any that are defined in `config.build`, where you can 1. `get_val_at_index` is a [helper function](#helper-functions); read more below! -1. Checking the `base_tech` of a technology (`storage`, `supply`, etc.) or its inheritance chain (if using `tech_groups` and the `inherit` parameter). +1. Checking the `base_tech` of a technology (`storage`, `supply`, etc.) or its inheritance chain (if using `templates` and the `template` parameter). ??? example "Examples" - If you want to create a decision variable across only `storage` technologies, you would include `base_tech=storage`. - - If you want to apply a constraint across only your own `rooftop_supply` technologies (e.g., you have defined `rooftop_supply` in `tech_groups` and your technologies `pv` and `solar_thermal` define `#!yaml inherit: rooftop_supply`), you would include `inheritance(rooftop_supply)`. - Note that `base_tech=...` is a simple check for the given value of `base_tech`, while `inheritance()` is a helper function ([see below](#helper-functions)) which can deal with the fact that intermediate groups may be present, e.g. `pv` might inherit from `rooftop_supply` which in turn might inherit from `electricity_supply`. + - If you want to apply a constraint across only your own `rooftop_supply` technologies (e.g., you have defined `rooftop_supply` in `templates` and your technologies `pv` and `solar_thermal` define `#!yaml template: rooftop_supply`), you would include `inheritance(rooftop_supply)`. + Note that `base_tech=...` is a simple check for the given value of `base_tech`, while `inheritance()` is a helper function ([see below](#helper-functions)) which can deal with finding techs/nodes using the same template, e.g. `pv` might inherit the `rooftop_supply` template which in turn might inherit the template `electricity_supply`. 1. Subsetting a set. The sets available to subset are always [`nodes`, `techs`, `carriers`] + any additional sets defined by you in [`foreach`](#foreach-lists). @@ -118,23 +118,23 @@ Some of these helper functions require a good understanding of their functionali ### inheritance -using `inheritance(...)` in a `where` string allows you to grab a subset of technologies that all share the same [`tech_group`](../creating/groups.md) in the technology's `inherit` key. -If a `tech_group` also inherits from another `tech_group` (chained inheritance), you will get all `techs` that are children along that inheritance chain. +using `inheritance(...)` in a `where` string allows you to grab a subset of technologies / nodes that all share the same [`template`](../creating/templates.md) in the technology's / node's `template` key. +If a `template` also inherits from another `template` (chained inheritance), you will get all `techs`/`nodes` that are children along that inheritance chain. So, for the definition: ```yaml -tech_groups: +templates: techgroup1: - inherit: techgroup2 + template: techgroup2 flow_cap_max: 10 techgroup2: base_tech: supply techs: tech1: - inherit: techgroup1 + template: techgroup1 tech2: - inherit: techgroup2 + template: techgroup2 ``` `inheritance(techgroup1)` will give the `[tech1]` subset and `inheritance(techgroup2)` will give the `[tech1, tech2]` subset. diff --git a/mkdocs.yml b/mkdocs.yml index b4e74cdd..1ea0bbad 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,6 @@ site_name: Calliope -site_url: https://calliope.readthedocs.io/ +# FIXME: `en/latest` should be changed to `en/stable` once there is a stable mkdocs based release +site_url: https://calliope.readthedocs.io/en/latest/ copyright: Copyright © since 2013 Calliope contributors (Apache 2.0 licensed) repo_url: https://github.com/calliope-project/calliope hooks: @@ -24,6 +25,9 @@ theme: - navigation.top - content.code.copy - content.code.annotate + - search.suggest + - search.highlight + - search.share logo: img/logo-bw.png favicon: img/favicon.ico extra_css: @@ -100,7 +104,7 @@ nav: - Model configuration: creating/config.md - Technologies: creating/techs.md - Nodes: creating/nodes.md - - Inheriting from technology and node groups: creating/groups.md + - Inheriting from templates: creating/templates.md - Indexed parameters: creating/parameters.md - Loading tabular data: creating/data_sources.md - Scenarios and overrides: creating/scenarios.md diff --git a/src/calliope/_version.py b/src/calliope/_version.py index d825d0ab..5e682eb3 100644 --- a/src/calliope/_version.py +++ b/src/calliope/_version.py @@ -1 +1 @@ -__version__ = "0.7.0.dev3" +__version__ = "0.7.0.dev5" diff --git a/src/calliope/backend/backend_model.py b/src/calliope/backend/backend_model.py index 8c53a22d..ef1f122b 100644 --- a/src/calliope/backend/backend_model.py +++ b/src/calliope/backend/backend_model.py @@ -1056,13 +1056,14 @@ def _get_variable_bound( name, f"Applying bound according to the {bound} parameter values.", ) - bound_array = self.get_parameter(bound).copy() + bound_array = self.get_parameter(bound) fill_na = bound_array.attrs.get("default", fill_na) references.add(bound) else: bound_array = xr.DataArray(bound) - bound_array.attrs = {} - return bound_array.fillna(fill_na) + filled_bound_array = bound_array.fillna(fill_na) + filled_bound_array.attrs = {} + return filled_bound_array @contextmanager def _datetime_as_string(self, data: xr.DataArray | xr.Dataset) -> Iterator: diff --git a/src/calliope/backend/helper_functions.py b/src/calliope/backend/helper_functions.py index 90924b95..1424cf29 100644 --- a/src/calliope/backend/helper_functions.py +++ b/src/calliope/backend/helper_functions.py @@ -163,7 +163,7 @@ def _listify(self, vals: list[str] | str) -> list[str]: class Inheritance(ParsingHelperFunction): - """Find all nodes / techs that inherit from a node / tech group.""" + """Find all nodes / techs that inherit from a template.""" #: ALLOWED_IN = ["where"] @@ -183,9 +183,9 @@ def as_math_string( # noqa: D102, override def as_array( self, *, nodes: str | None = None, techs: str | None = None ) -> xr.DataArray: - """Find all technologies and/or nodes which inherit from a particular technology or node group. + """Find all technologies and/or nodes which inherit from a particular template. - The group items being referenced must be defined by the user in `node_groups`/`tech_groups`. + The group items being referenced must be defined by the user in `templates`. Args: nodes (str | None, optional): group name to search for inheritance of on the `nodes` dimension. Default is None. @@ -197,28 +197,27 @@ def as_array( Examples: With: ```yaml - node_groups: + templates: foo: available_area: 1 - tech_groups: bar: flow_cap_max: 1 baz: - inherits: bar + template: bar flow_out_eff: 0.5 nodes: node_1: - inherits: foo + template: foo techs: {tech_1, tech_2} node_2: techs: {tech_1, tech_2} techs: tech_1: ... - inherits: bar + template: bar tech_2: ... - inherits: baz + template: baz ``` >>> inheritance(nodes=foo) diff --git a/src/calliope/backend/latex_backend_model.py b/src/calliope/backend/latex_backend_model.py index 533bf958..c0eda0d8 100644 --- a/src/calliope/backend/latex_backend_model.py +++ b/src/calliope/backend/latex_backend_model.py @@ -90,10 +90,18 @@ class LatexBackendModel(backend_model.BackendModelGenerator): {{ equation.description }} {% endif %} - {% if equation.references %} + {% if equation.used_in %} **Used in**: - {% for ref in equation.references %} + {% for ref in equation.used_in %} + + * {{ ref }} + {% endfor %} + {% endif %} + {% if equation.uses %} + + **Uses**: + {% for ref in equation.uses %} * {{ ref }} {% endfor %} @@ -155,10 +163,19 @@ class LatexBackendModel(backend_model.BackendModelGenerator): {{ equation.description }} {% endif %} - {% if equation.references %} + {% if equation.used_in %} \textbf{Used in}: - {% for ref in equation.references %} + {% for ref in equation.used_in %} + \begin{itemize} + \item {{ ref }} + \end{itemize} + {% endfor %} + {% endif %} + {% if equation.uses %} + + \textbf{Uses}: + {% for ref in equation.uses %} \begin{itemize} \item {{ ref }} \end{itemize} @@ -212,12 +229,28 @@ class LatexBackendModel(backend_model.BackendModelGenerator): {{ equation.description }} {% endif %} - {% if equation.references %} + {% if equation.used_in %} + {% if mkdocs_features %} + ??? info "Used in" + {% else %} **Used in**: + {% endif %} + + {% for ref in equation.used_in %} + {{ " " if mkdocs_features else "" }}* [{{ ref }}](#{{ ref }}) + {% endfor %} + {% endif %} + {% if equation.uses %} - {% for ref in equation.references %} - * [{{ ref }}](#{{ ref }}) + {% if mkdocs_features %} + ??? info "Uses" + {% else %} + **Uses**: + {% endif %} + + {% for ref in equation.uses %} + {{ " " if mkdocs_features else "" }}* [{{ ref }}](#{{ ref }}) {% endfor %} {% endif %} {% if equation.unit is not none %} @@ -229,7 +262,7 @@ class LatexBackendModel(backend_model.BackendModelGenerator): **Default**: {{ equation.default }} {% endif %} {% if equation.expression != "" %} - {% if mkdocs_tabbed and yaml_snippet is not none%} + {% if mkdocs_features and yaml_snippet is not none%} === "Math" @@ -398,7 +431,7 @@ def _variable_setter(where: xr.DataArray, references: set) -> xr.DataArray: self._update_references(name, bound_refs.difference(name)) self._generate_math_string( - parsed_component, var_da, equations=[lb, ub], sense=r"\forall" + domain + parsed_component, var_da, equations=[lb, ub], sense=r"\in" + domain ) def add_objective( # noqa: D102, override @@ -440,36 +473,50 @@ def delete_component( # noqa: D102, override del self._dataset[key] def generate_math_doc( - self, format: ALLOWED_MATH_FILE_FORMATS = "tex", mkdocs_tabbed: bool = False + self, format: ALLOWED_MATH_FILE_FORMATS = "tex", mkdocs_features: bool = False ) -> str: """Generate the math documentation by embedding LaTeX math in a template. Args: format (Literal["tex", "rst", "md"]): The built LaTeX math will be embedded in a document of the given format (tex=LaTeX, rst=reStructuredText, md=Markdown). Defaults to "tex". - mkdocs_tabbed (bool, optional): If True and format is `md`, the equations will be on a tab and the original YAML math definition will be on another tab. + mkdocs_features (bool, optional): + If True and format is `md`, then: + - the equations will be on a tab and the original YAML math definition will be on another tab; + - the equation cross-references will be given in a drop-down list. Returns: str: Generated math documentation. """ - if mkdocs_tabbed and format != "md": + if mkdocs_features and format != "md": raise ModelError( - "Cannot use MKDocs tabs when writing math to a non-Markdown file format." + "Cannot use MKDocs features when writing math to a non-Markdown file format." ) doc_template = self.FORMAT_STRINGS[format] + uses = { + name: set( + other + for other, da_other in self._dataset.data_vars.items() + if name in da_other.attrs.get("references", set()) + ) + for name in self._dataset.data_vars + } components = { objtype: [ { "expression": da.attrs.get("math_string", ""), "name": name, "description": da.attrs.get("description", None), - "references": list(da.attrs.get("references", set())), + "used_in": sorted( + list(da.attrs.get("references", set()) - set([name])) + ), + "uses": sorted(list(uses[name] - set([name]))), "default": da.attrs.get("default", None), "unit": da.attrs.get("unit", None), "yaml_snippet": da.attrs.get("yaml_snippet", None), } - for name, da in getattr(self, objtype).data_vars.items() + for name, da in sorted(getattr(self, objtype).data_vars.items()) if "math_string" in da.attrs or (objtype == "parameters" and da.attrs["references"]) ] @@ -486,7 +533,7 @@ def generate_math_doc( if "parameters" in components and not components["parameters"]: del components["parameters"] return self._render( - doc_template, mkdocs_tabbed=mkdocs_tabbed, components=components + doc_template, mkdocs_features=mkdocs_features, components=components ) def _add_latex_strings( diff --git a/src/calliope/backend/pyomo_backend_model.py b/src/calliope/backend/pyomo_backend_model.py index 15f204b7..3c500a71 100644 --- a/src/calliope/backend/pyomo_backend_model.py +++ b/src/calliope/backend/pyomo_backend_model.py @@ -211,7 +211,7 @@ def get_parameter( # noqa: D102, override ) orig_dtype = parameter.original_dtype self.log("parameters", name, f"Converting Pyomo object to {orig_dtype} dtype.") - return param_as_vals.astype(orig_dtype) + return param_as_vals.astype(orig_dtype).where(param_as_vals.notnull()) @overload def get_constraint( # noqa: D102, override @@ -688,7 +688,7 @@ def _from_pyomo_expr(self, expr_da: xr.DataArray, eval_body: bool) -> xr.DataArr expr = expr_da.astype(str) else: expr = expr_da.astype(str) - return expr.where(expr.notnull()) + return expr.where(expr_da.notnull()) @staticmethod def _from_pyomo_param(val: ObjParameter | ObjVariable | float) -> Any: diff --git a/src/calliope/config/config_schema.yaml b/src/calliope/config/config_schema.yaml index d5d2dec2..17b30812 100644 --- a/src/calliope/config/config_schema.yaml +++ b/src/calliope/config/config_schema.yaml @@ -221,18 +221,10 @@ properties: additionalProperties: false patternProperties: *nested_pattern - tech_groups: + templates: type: [object, "null"] description: >- - Abstract technology definitions from which techs can `inherit`. - See the model definition schema for more guidance on content. - additionalProperties: false - patternProperties: *nested_pattern - - node_groups: - type: [object, "null"] - description: >- - Abstract technology definitions from which nodes can `inherit`. + Abstract technology/node templates from which techs/nodes can `inherit`. See the model definition schema for more guidance on content. additionalProperties: false patternProperties: *nested_pattern diff --git a/src/calliope/config/model_data_checks.yaml b/src/calliope/config/model_data_checks.yaml index 051ebed6..d8491743 100644 --- a/src/calliope/config/model_data_checks.yaml +++ b/src/calliope/config/model_data_checks.yaml @@ -26,8 +26,8 @@ fail: - where: carrier_export and not any(carrier_out, over=nodes) message: "Export carriers must be one of the technology outflow carriers." - - where: storage_initial>1 - message: "storage_initial is a fraction; values larger than 1 are not allowed." + - where: storage_initial<0 OR storage_initial>1 + message: "storage_initial is a fraction, requiring values within the interval [0, 1]." - where: integer_dispatch=True AND NOT cap_method=integer message: Cannot use the integer `integer_dispatch` unless the technology is using an integer unit capacities (`cap_method=integer`). diff --git a/src/calliope/config/model_def_schema.yaml b/src/calliope/config/model_def_schema.yaml index b9fd2ab9..3f536642 100644 --- a/src/calliope/config/model_def_schema.yaml +++ b/src/calliope/config/model_def_schema.yaml @@ -407,9 +407,16 @@ properties: x-type: float title: Minimum rated flow capacity. description: >- - Limits `flow_cap` to a minimum. - NOTE: this will _force_ `flow_cap` to a minimum value unless `cap_method` is set to `integer`. - If `cap_method=integer`, this will be scaled by the number of integer units of a technology purchased. + This acts as lower limit (lower bound) for flow_cap. + If `cap_method` = integer, this will be scaled by the number of integer units of a technology purchased. + + !!! note + If `cap_method` = continuous (which is set by default), then setting `flow_cap_min` to any value greater than 0 forces the model to always respect that limit, which in turn forces investment into the technology. + Use this to enforce given investment plans, or to account for existing technologies that cannot be decommissioned. + + An example: If `cap_method` = continuous and `flow_cap_min` = 100, then flow_cap has to be at least 100. + This means that going to 0 is not possible, unless you switch to `cap_method` = integer (see [cap_method][]). + x-unit: power or $\frac{\text{power}}{\text{unit}}$. flow_cap_min_systemwide: @@ -675,7 +682,7 @@ properties: x-type: str title: Sink unit description: >- - Sets the unit of `Sink` to either `absolute` x-unit: energy), `per_area` x-unit: energy/area), or `per_cap` x-unit: energy/power). + Sets the unit of `Sink` to either `absolute` (unit: `energy`), `per_area` (unit: `energy/area`), or `per_cap` (unit: `energy/power`). `per_area` uses the `area_use` decision variable to scale the sink while `per_cap` uses the `flow_cap` decision variable. enum: [absolute, per_area, per_cap] @@ -687,7 +694,7 @@ properties: title: Minimum bound on sink. description: >- Minimum sink use to remove a carrier from the system (e.g., electricity demand, transport distance). - Unit dictated by `source_unit`. + Unit dictated by `sink_unit`. sink_use_max: $ref: "#/$defs/TechParamNullNumber" @@ -697,7 +704,7 @@ properties: title: Maximum bound on sink. description: >- Maximum sink use to remove a carrier from the system (e.g., electricity demand, transport distance). - Unit dictated by `source_unit`. + Unit dictated by `sink_unit`. sink_use_equals: $ref: "#/$defs/TechParamNullNumber" @@ -707,7 +714,7 @@ properties: title: Required sink use. description: >- Required amount of carrier removal from the system (e.g., electricity demand, transport distance). - Unit dictated by `source_unit`. + Unit dictated by `sink_unit`. source_unit: type: string @@ -716,7 +723,7 @@ properties: x-type: str title: Source unit description: >- - Sets the unit of `Source` to either `absolute` (e.g. kWh), `per_area` (e.g. kWh/m2), or `per_cap` (e.g. kWh/kW). + Sets the unit of `Source` to either `absolute` (unit: `energy`), `per_area` (unit: `energy/area`), or `per_cap` (unit: `energy/power`). `per_area` uses the `area_use` decision variable to scale the source while `per_cap` uses the `flow_cap` decision variable. enum: [absolute, per_area, per_cap] @@ -903,7 +910,7 @@ properties: x-type: float title: Fractional annual O&M costs. description: >- - Add an additional cost to total investment costs (except `cost_om_annual`) that is a fraction of that total. + Add a fraction of the sum of all investment costs except `cost_om_annual` as an additional cost, to represent fixed annual O&M costs. Warning: the sum of all investment costs includes not just those associated with `flow_cap` but also others like those associated with `area_use`! x-unit: fraction / total investment. cost_flow_in: @@ -1008,4 +1015,4 @@ properties: x-type: float minimum: 0 default: .inf - x-resample_method: mean \ No newline at end of file + x-resample_method: mean diff --git a/src/calliope/config/protected_parameters.yaml b/src/calliope/config/protected_parameters.yaml index b2554723..8ed48d94 100644 --- a/src/calliope/config/protected_parameters.yaml +++ b/src/calliope/config/protected_parameters.yaml @@ -4,8 +4,6 @@ active: >- YAML model definition. definition_matrix: >- `definition_matrix` is a protected array. - It will be generated internally based on the values you assign to - the `carrier_in` and `carrier_out` parameters. -inherit: >- - Technology/Node inheritance (`inherit`) can only be used in the - YAML model definition. + It will be generated internally based on the values you assign to the `carrier_in` and `carrier_out` parameters. +template: >- + Technology/Node template inheritance (`template`) can only be used in the YAML model definition. diff --git a/src/calliope/example_models/national_scale/model_config/locations.yaml b/src/calliope/example_models/national_scale/model_config/locations.yaml index d0039f24..f5efad47 100644 --- a/src/calliope/example_models/national_scale/model_config/locations.yaml +++ b/src/calliope/example_models/national_scale/model_config/locations.yaml @@ -21,24 +21,24 @@ nodes: battery: region1_1: - inherit: csp_regions + template: csp_regions latitude: 41 longitude: -2 region1_2: - inherit: csp_regions + template: csp_regions latitude: 39 longitude: -1 region1_3: - inherit: csp_regions + template: csp_regions latitude: 39 longitude: -2 # --8<-- [end:other-locs] -# --8<-- [start:node-groups] -node_groups: +# --8<-- [start:templates] +templates: csp_regions: techs: csp: -# --8<-- [end:node-groups] +# --8<-- [end:templates] diff --git a/src/calliope/example_models/national_scale/model_config/techs.yaml b/src/calliope/example_models/national_scale/model_config/techs.yaml index 71a5b78c..d0b481e4 100644 --- a/src/calliope/example_models/national_scale/model_config/techs.yaml +++ b/src/calliope/example_models/national_scale/model_config/techs.yaml @@ -5,7 +5,7 @@ # Note: --8<--start:'' and --8<--end:'' is used in tutorial documentation only # --8<-- [start:cost-dim-setter] -tech_groups: +templates: cost_dim_setter: cost_flow_cap: data: null @@ -56,7 +56,7 @@ techs: name: "Combined cycle gas turbine" color: "#E37A72" base_tech: supply - inherit: cost_dim_setter + template: cost_dim_setter carrier_out: power flow_out_eff: 0.5 flow_cap_max: 40000 # kW @@ -73,7 +73,7 @@ techs: name: "Concentrating solar power" color: "#F9CF22" base_tech: supply - inherit: cost_dim_setter + template: cost_dim_setter carrier_out: power source_unit: per_area include_storage: True @@ -101,7 +101,7 @@ techs: name: "Battery storage" color: "#3B61E3" base_tech: storage - inherit: cost_dim_setter + template: cost_dim_setter carrier_in: power carrier_out: power flow_cap_max: 1000 # kW @@ -139,7 +139,7 @@ techs: name: "AC power transmission" color: "#8465A9" base_tech: transmission - inherit: cost_dim_setter + template: cost_dim_setter carrier_in: power carrier_out: power flow_out_eff: 0.85 @@ -151,14 +151,14 @@ techs: region1_to_region1_1: from: region1 to: region1_1 - inherit: free_transmission + template: free_transmission region1_to_region1_2: from: region1 to: region1_2 - inherit: free_transmission + template: free_transmission region1_to_region1_3: from: region1 to: region1_3 - inherit: free_transmission + template: free_transmission # --8<-- [end:transmission] diff --git a/src/calliope/example_models/national_scale/scenarios.yaml b/src/calliope/example_models/national_scale/scenarios.yaml index 214fd1b3..a0763950 100644 --- a/src/calliope/example_models/national_scale/scenarios.yaml +++ b/src/calliope/example_models/national_scale/scenarios.yaml @@ -51,7 +51,7 @@ overrides: # FIXME: replace group constraint in SPORES mode # group_constraints: # systemwide_cost_max.cost_max.monetary: 1e10 # very large, non-infinite value - tech_groups: + templates: cost_dim_setter: cost_flow_cap: index: [monetary, spores_score] @@ -140,7 +140,7 @@ overrides: name: "Cold fusion" color: "#233B39" base_tech: supply - inherit: cost_dim_setter + template: cost_dim_setter carrier_out: power flow_cap_max: 10000 lifetime: 50 diff --git a/src/calliope/example_models/urban_scale/model_config/techs.yaml b/src/calliope/example_models/urban_scale/model_config/techs.yaml index da45be6c..50e29c9b 100644 --- a/src/calliope/example_models/urban_scale/model_config/techs.yaml +++ b/src/calliope/example_models/urban_scale/model_config/techs.yaml @@ -5,19 +5,19 @@ # --8<-- [start:--8<-- [end:Note: ']' and ']' is used in tutorial documentation only # --8<-- [start:interest-rate-setter] -tech_groups: +templates: interest_rate_setter: cost_interest_rate: data: 0.10 index: monetary dims: costs # --8<-- [end:interest-rate-setter] -# --8<-- [start:transmission-tech-groups] +# --8<-- [start:transmission-templates] power_lines: name: "Electrical power distribution" color: "#6783E3" base_tech: transmission - inherit: interest_rate_setter + template: interest_rate_setter carrier_in: electricity carrier_out: electricity flow_cap_max: 2000 @@ -32,7 +32,7 @@ tech_groups: name: "District heat distribution" color: "#823739" base_tech: transmission - inherit: interest_rate_setter + template: interest_rate_setter carrier_in: heat carrier_out: heat flow_cap_max: 2000 @@ -42,7 +42,7 @@ tech_groups: data: 0.3 index: monetary dims: costs -# --8<-- [end:transmission-tech-groups] +# --8<-- [end:transmission-templates] techs: ##-GRID SUPPLY-## # --8<-- [start:supply] @@ -50,7 +50,7 @@ techs: name: "National grid import" color: "#C5ABE3" base_tech: supply - inherit: interest_rate_setter + template: interest_rate_setter carrier_out: electricity source_use_max: .inf flow_cap_max: 2000 @@ -68,7 +68,7 @@ techs: name: "Natural gas import" color: "#C98AAD" base_tech: supply - inherit: interest_rate_setter + template: interest_rate_setter carrier_out: gas source_use_max: .inf flow_cap_max: 2000 @@ -90,7 +90,7 @@ techs: color: "#F9D956" base_tech: supply carrier_out: electricity - inherit: interest_rate_setter + template: interest_rate_setter carrier_export: electricity source_unit: per_area area_use_per_flow_cap: 7 # 7m2 of panels needed to fit 1kWp of panels @@ -111,7 +111,7 @@ techs: name: "Natural gas boiler" color: "#8E2999" base_tech: conversion - inherit: interest_rate_setter + template: interest_rate_setter carrier_in: gas carrier_out: heat flow_cap_max: @@ -132,7 +132,7 @@ techs: name: "Combined heat and power" color: "#E4AB97" base_tech: conversion - inherit: interest_rate_setter + template: interest_rate_setter carrier_in: gas carrier_out: [electricity, heat] carrier_export: electricity @@ -177,27 +177,27 @@ techs: X1_to_X2: from: X1 to: X2 - inherit: power_lines + template: power_lines distance: 10 X1_to_X3: from: X1 to: X3 - inherit: power_lines + template: power_lines distance: 5 X1_to_N1: from: X1 to: N1 - inherit: heat_pipes + template: heat_pipes distance: 3 N1_to_X2: from: N1 to: X2 - inherit: heat_pipes + template: heat_pipes distance: 3 N1_to_X3: from: N1 to: X3 - inherit: heat_pipes + template: heat_pipes distance: 4 # --8<-- [end:transmission] diff --git a/src/calliope/example_models/urban_scale/scenarios.yaml b/src/calliope/example_models/urban_scale/scenarios.yaml index 3b217706..12d114cb 100644 --- a/src/calliope/example_models/urban_scale/scenarios.yaml +++ b/src/calliope/example_models/urban_scale/scenarios.yaml @@ -40,7 +40,7 @@ overrides: dims: costs # --8<-- [end:boiler] # --8<-- [start:heat_pipes] - tech_groups: + templates: heat_pipes: force_async_flow: true # --8<-- [end:heat_pipes] diff --git a/src/calliope/math/plan.yaml b/src/calliope/math/plan.yaml index b2f3c6bc..0ee9f7db 100644 --- a/src/calliope/math/plan.yaml +++ b/src/calliope/math/plan.yaml @@ -800,9 +800,9 @@ global_expressions: - expression: timestep_weights * ($cost_export + $cost_flow_out + $cost_flow_in) sub_expressions: cost_export: - - where: flow_export + - where: any(carrier_export, over=carriers) AND any(cost_export, over=carriers) expression: sum(cost_export * flow_export, over=carriers) - - where: NOT flow_export + - where: NOT (any(carrier_export, over=carriers) AND any(cost_export, over=carriers)) expression: "0" cost_flow_in: - where: "base_tech=supply" @@ -884,13 +884,13 @@ global_expressions: equations: - expression: > $annualisation_weight * ( - $depreciation_rate * ( + ($depreciation_rate + cost_om_annual_investment_fraction) * ( sum(default_if_empty(cost_investment_flow_cap, 0), over=carriers) + default_if_empty(cost_investment_storage_cap, 0) + default_if_empty(cost_investment_source_cap, 0) + default_if_empty(cost_investment_area_use, 0) + default_if_empty(cost_investment_purchase, 0) - ) * (1 + cost_om_annual_investment_fraction) + ) + sum(cost_om_annual * flow_cap, over=carriers) ) sub_expressions: diff --git a/src/calliope/postprocess/math_documentation.py b/src/calliope/postprocess/math_documentation.py index a37f73ce..ebfb3193 100644 --- a/src/calliope/postprocess/math_documentation.py +++ b/src/calliope/postprocess/math_documentation.py @@ -3,7 +3,7 @@ import logging import typing from pathlib import Path -from typing import Literal +from typing import Literal, overload from calliope.backend import ALLOWED_MATH_FILE_FORMATS, LatexBackendModel from calliope.model import Model @@ -39,10 +39,23 @@ def math(self): """Direct access to backend math.""" return self.backend.math + # Expecting string if not giving filename. + @overload + def write( + self, + filename: Literal[None] = None, + mkdocs_features: bool = False, + format: ALLOWED_MATH_FILE_FORMATS | None = None, + ) -> str: ... + + # Expecting None (and format arg is not needed) if giving filename. + @overload + def write(self, filename: str | Path, mkdocs_features: bool = False) -> None: ... + def write( self, filename: str | Path | None = None, - mkdocs_tabbed: bool = False, + mkdocs_features: bool = False, format: ALLOWED_MATH_FILE_FORMATS | None = None, ) -> str | None: """Write model documentation. @@ -51,9 +64,10 @@ def write( filename (str | Path | None, optional): If given, will write the built mathematical formulation to a file with the given extension as the file format. Defaults to None. - mkdocs_tabbed (bool, optional): - If True and Markdown docs are being generated, the equations will be on - a tab and the original YAML math definition will be on another tab. + mkdocs_features (bool, optional): + If True and Markdown docs are being generated, then: + - the equations will be on a tab and the original YAML math definition will be on another tab; + - the equation cross-references will be given in a drop-down list. Defaults to False. format (ALLOWED_MATH_FILE_FORMATS | None, optional): Not required if filename is given (as the format will be automatically inferred). @@ -78,7 +92,7 @@ def write( raise ValueError( f"Math documentation format must be one of {allowed_formats}, received `{format}`" ) - populated_doc = self.backend.generate_math_doc(format, mkdocs_tabbed) + populated_doc = self.backend.generate_math_doc(format, mkdocs_features) if filename is None: return populated_doc diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index ffd923ef..442c6226 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -37,8 +37,7 @@ class ModelDefinition(TypedDict): techs: AttrDict nodes: AttrDict - tech_groups: NotRequired[AttrDict] - node_groups: NotRequired[AttrDict] + templates: NotRequired[AttrDict] parameters: NotRequired[AttrDict] @@ -83,7 +82,7 @@ def __init__( Args: model_config (dict): Model initialisation configuration (i.e., `config.init`). - model_definition (ModelDefinition): Definition of model nodes and technologies, and their potential inheritance `groups`. + model_definition (ModelDefinition): Definition of model nodes and technologies, and their potential `templates`. data_sources (list[data_sources.DataSource]): Pre-loaded data sources that will be used to initialise the dataset before handling definitions given in `model_definition`. attributes (dict): Attributes to attach to the model Dataset. param_attributes (dict[str, dict]): Attributes to attach to the generated model DataArrays. @@ -149,7 +148,7 @@ def init_from_data_sources(self, data_sources: list[data_sources.DataSource]): def add_node_tech_data(self): """For each node, extract technology definitions and node-level parameters and convert them to arrays. - The node definition will first be updated according to any defined inheritance (via `inherit`), + The node definition will first be updated according to any defined inheritance (via `template`), before processing each defined tech (which will also be updated according to its inheritance tree). Node and tech definitions will be validated against the model definition schema here. @@ -399,7 +398,7 @@ def _get_relevant_node_refs(self, techs_dict: AttrDict, node: str) -> list[str]: if "base_tech" in tech_dict.keys(): raise exceptions.ModelError( f"(nodes, {node}), (techs, {tech_name}) | Defining a technology `base_tech` at a node is not supported; " - "limit yourself to defining this parameter within `techs` or `tech_groups`" + "limit yourself to defining this parameter within `techs` or `templates`" ) refs.update(tech_dict.keys()) @@ -510,7 +509,7 @@ def _inherit_defs( ) -> AttrDict: """For a set of node/tech definitions, climb the inheritance tree to build a final definition dictionary. - For `techs` at `nodes`, the first step is to inherit the technology definition from `techs`, _then_ to climb `inherit` references. + For `techs` at `nodes`, the first step is to inherit the technology definition from `techs`, _then_ to climb `template` references. Base definitions will take precedence over inherited ones and more recent inherited definitions will take precedence over older ones. @@ -558,7 +557,7 @@ def _inherit_defs( item_base_def.union(item_def, allow_override=True) else: item_base_def = item_def - updated_item_def, inheritance = self._climb_inheritance_tree( + updated_item_def, inheritance = self._climb_template_tree( item_base_def, dim_name, item_name ) @@ -571,47 +570,44 @@ def _inherit_defs( if inheritance is not None: updated_item_def[f"{dim_name}_inheritance"] = ",".join(inheritance) - del updated_item_def["inherit"] + del updated_item_def["template"] updated_defs[item_name] = updated_item_def return updated_defs - def _climb_inheritance_tree( + def _climb_template_tree( self, dim_item_dict: AttrDict, dim_name: Literal["nodes", "techs"], item_name: str, inheritance: list | None = None, ) -> tuple[AttrDict, list | None]: - """Follow the `inherit` references from `nodes` to `node_groups` / from `techs` to `tech_groups`. + """Follow the `template` references from `nodes` / `techs` to `templates`. - Abstract group definitions (those in `node_groups`/`tech_groups`) can inherit each other, but `nodes`/`techs` cannot. + Abstract template definitions (those in `templates`) can inherit each other, but `nodes`/`techs` cannot. - This function will be called recursively until a definition dictionary without `inherit` is reached. + This function will be called recursively until a definition dictionary without `template` is reached. Args: - dim_item_dict (AttrDict): - Dictionary (possibly) containing `inherit`. If it doesn't contain `inherit`, the climbing stops here. + dim_item_dict (AttrDict): Dictionary (possibly) containing `template`. dim_name (Literal[nodes, techs]): The name of the dimension we're working with, so that we can access the correct `_groups` definitions. item_name (str): The current position in the inheritance tree. inheritance (list | None, optional): A list of items that have been inherited (starting with the oldest). - If the first `dim_item_dict` does not contain `inherit`, this will remain as None. + If the first `dim_item_dict` does not contain `template`, this will remain as None. Defaults to None. Raises: - KeyError: Must inherit from a named group item in `node_groups` (for `nodes`) and `tech_groups` (for `techs`) + KeyError: Must inherit from a named template item in `templates`. Returns: tuple[AttrDict, list | None]: Definition dictionary with inherited data and a list of the inheritance tree climbed to get there. """ - to_inherit = dim_item_dict.get("inherit", None) - dim_groups = AttrDict( - self.model_definition.get(f"{dim_name.removesuffix('s')}_groups", {}) - ) + to_inherit = dim_item_dict.get("template", None) + dim_groups = AttrDict(self.model_definition.get("templates", {})) if to_inherit is None: if dim_name == "techs" and item_name in self.tech_data_from_sources: _data_source_dict = deepcopy(self.tech_data_from_sources[item_name]) @@ -620,10 +616,10 @@ def _climb_inheritance_tree( updated_dim_item_dict = dim_item_dict elif to_inherit not in dim_groups: raise KeyError( - f"({dim_name}, {item_name}) | Cannot find `{to_inherit}` in inheritance tree." + f"({dim_name}, {item_name}) | Cannot find `{to_inherit}` in template inheritance tree." ) else: - base_def_dict, inheritance = self._climb_inheritance_tree( + base_def_dict, inheritance = self._climb_template_tree( dim_groups[to_inherit], dim_name, to_inherit, inheritance ) updated_dim_item_dict = deepcopy(base_def_dict) diff --git a/tests/common/lp_files/cost_var.lp b/tests/common/lp_files/cost_var.lp new file mode 100644 index 00000000..b65a2c24 --- /dev/null +++ b/tests/common/lp_files/cost_var.lp @@ -0,0 +1,43 @@ +\* Source Pyomo model name=None *\ + +min +objectives(foo)(0): ++0.1 variables(flow_out)(a__test_conversion__heat__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_conversion__heat__2005_01_01_01_00) ++1.0 variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_00_00) ++2.0 variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_00_00) ++4.0 variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_00_00) ++1.0 variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_01_00) ++2.0 variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_01_00) ++4.0 variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_01_00) ++0.1 variables(flow_out)(a__test_supply_coal__coal__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_supply_coal__coal__2005_01_01_01_00) ++0.1 variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_01_00) ++0.1 variables(flow_out)(a__test_supply_gas__gas__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_supply_gas__gas__2005_01_01_01_00) + +s.t. + +c_e_ONE_VAR_CONSTANT: ++1 ONE_VAR_CONSTANT += 1 + +bounds + 1 <= ONE_VAR_CONSTANT <= 1 + 0 <= variables(flow_out)(a__test_conversion__heat__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion__heat__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_00_00) <= +inf + 0 <= variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_01_00) <= +inf + 0 <= variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_coal__coal__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_coal__coal__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_gas__gas__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_gas__gas__2005_01_01_01_00) <= +inf +end + diff --git a/tests/common/lp_files/cost_var_with_export.lp b/tests/common/lp_files/cost_var_with_export.lp new file mode 100644 index 00000000..9c4c1b7b --- /dev/null +++ b/tests/common/lp_files/cost_var_with_export.lp @@ -0,0 +1,51 @@ +\* Source Pyomo model name=None *\ + +min +objectives(foo)(0): ++0.1 variables(flow_out)(a__test_conversion__heat__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_conversion__heat__2005_01_01_01_00) ++1.0 variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_00_00) ++2.0 variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_00_00) ++3.0 variables(flow_export)(a__test_conversion_plus__heat__2005_01_01_00_00) ++4.0 variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_00_00) ++1.0 variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_01_00) ++2.0 variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_01_00) ++3.0 variables(flow_export)(a__test_conversion_plus__heat__2005_01_01_01_00) ++4.0 variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_01_00) ++0.1 variables(flow_out)(a__test_supply_coal__coal__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_supply_coal__coal__2005_01_01_01_00) ++5.0 variables(flow_export)(a__test_supply_elec__electricity__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_00_00) ++5.0 variables(flow_export)(a__test_supply_elec__electricity__2005_01_01_01_00) ++0.1 variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_01_00) ++0.1 variables(flow_out)(a__test_supply_gas__gas__2005_01_01_00_00) ++0.1 variables(flow_out)(a__test_supply_gas__gas__2005_01_01_01_00) + +s.t. + +c_e_ONE_VAR_CONSTANT: ++1 ONE_VAR_CONSTANT += 1 + +bounds + 1 <= ONE_VAR_CONSTANT <= 1 + 0 <= variables(flow_out)(a__test_conversion__heat__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion__heat__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_00_00) <= +inf + 0 <= variables(flow_export)(a__test_conversion_plus__heat__2005_01_01_00_00) <= +inf + 0 <= variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_conversion_plus__heat__2005_01_01_01_00) <= +inf + 0 <= variables(flow_export)(a__test_conversion_plus__heat__2005_01_01_01_00) <= +inf + 0 <= variables(flow_in)(a__test_conversion_plus__gas__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_coal__coal__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_coal__coal__2005_01_01_01_00) <= +inf + 0 <= variables(flow_export)(a__test_supply_elec__electricity__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_00_00) <= +inf + 0 <= variables(flow_export)(a__test_supply_elec__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_elec__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_gas__gas__2005_01_01_00_00) <= +inf + 0 <= variables(flow_out)(a__test_supply_gas__gas__2005_01_01_01_00) <= +inf +end + diff --git a/tests/common/test_model/model.yaml b/tests/common/test_model/model.yaml index 88380b76..aa905680 100644 --- a/tests/common/test_model/model.yaml +++ b/tests/common/test_model/model.yaml @@ -32,17 +32,13 @@ data_sources: parameters: sink_use_equals techs: test_demand_elec -node_groups: - init_nodes: - techs.test_demand_elec: - nodes: - a.inherit: init_nodes - b.inherit: init_nodes + a.template: init_nodes + b.template: init_nodes techs: test_supply_gas: - inherit: test_controller + template: test_controller name: Supply tech carrier_out: gas base_tech: supply @@ -51,7 +47,7 @@ techs: flow_out_eff: 0.9 test_supply_elec: - inherit: test_controller + template: test_controller name: Supply tech carrier_out: electricity base_tech: supply @@ -60,7 +56,7 @@ techs: flow_out_eff: 0.9 test_supply_coal: - inherit: test_controller + template: test_controller name: Supply tech carrier_out: coal base_tech: supply @@ -69,7 +65,7 @@ techs: flow_out_eff: 0.9 test_supply_plus: - inherit: test_controller + template: test_controller name: Supply tech with storage carrier_out: electricity base_tech: supply @@ -82,7 +78,7 @@ techs: include_storage: true test_storage: - inherit: test_controller + template: test_controller name: Storage tech carrier_out: electricity carrier_in: electricity @@ -94,7 +90,7 @@ techs: storage_loss: 0.01 test_conversion: - inherit: test_controller + template: test_controller name: Conversion tech carrier_in: gas carrier_out: heat @@ -106,7 +102,7 @@ techs: flow_out_eff: 0.9 test_conversion_plus: - inherit: test_controller + template: test_controller name: Conversion tech with linked carriers out carrier_in: gas carrier_out: [electricity, heat] @@ -118,7 +114,7 @@ techs: heat_to_power_ratio: 0.8 test_chp: - inherit: test_controller + template: test_controller name: Conversion tech with unlinked carriers out carrier_in: gas carrier_out: [electricity, heat] @@ -149,23 +145,23 @@ techs: sink_use_max: 10 test_link_a_b_elec: - inherit: test_transmission_elec + template: test_transmission_elec distance: 1 test_link_a_b_heat: - inherit: test_transmission_heat + template: test_transmission_heat distance: 2 -tech_groups: +templates: test_controller: {} test_transmission: from: a to: b - inherit: test_controller + template: test_controller test_transmission_elec: - inherit: test_transmission + template: test_transmission name: Transmission elec tech carrier_in: electricity carrier_out: electricity @@ -173,9 +169,12 @@ tech_groups: flow_cap_max: 10 test_transmission_heat: - inherit: test_transmission + template: test_transmission name: Transmission heat tech carrier_in: heat carrier_out: heat base_tech: transmission flow_cap_max: 5 + + init_nodes: + techs.test_demand_elec: diff --git a/tests/common/test_model/scenarios.yaml b/tests/common/test_model/scenarios.yaml index 3f692274..f0f7f78b 100644 --- a/tests/common/test_model/scenarios.yaml +++ b/tests/common/test_model/scenarios.yaml @@ -81,7 +81,7 @@ overrides: index: monetary dims: costs - tech_groups.test_transmission.active: false + templates.test_transmission.active: false supply_purchase: nodes: @@ -99,7 +99,7 @@ overrides: index: monetary dims: costs - tech_groups.test_transmission.active: false + templates.test_transmission.active: false supply_milp: techs: @@ -114,7 +114,7 @@ overrides: techs: test_supply_elec: b.active: false - tech_groups.test_transmission.active: false + templates.test_transmission.active: false supply_export: techs: @@ -177,7 +177,7 @@ overrides: test_conversion: test_conversion_plus: - tech_groups.test_transmission.active: false + templates.test_transmission.active: false conversion_plus_milp: data_sources: @@ -206,7 +206,7 @@ overrides: test_supply_gas: test_conversion_plus: - tech_groups.test_transmission.active: false + templates.test_transmission.active: false conversion_milp: data_sources: @@ -231,7 +231,7 @@ overrides: test_supply_gas: test_conversion: - tech_groups.test_transmission.active: false + templates.test_transmission.active: false conversion_plus_purchase: data_sources: @@ -257,7 +257,7 @@ overrides: index: monetary dims: costs - tech_groups.test_transmission.active: false + templates.test_transmission.active: false simple_conversion_plus: data_sources: @@ -278,7 +278,7 @@ overrides: test_supply_coal: test_conversion_plus: - tech_groups.test_transmission.active: false + templates.test_transmission.active: false simple_chp: data_sources: @@ -298,7 +298,7 @@ overrides: test_supply_gas: test_chp: - tech_groups.test_transmission.active: false + templates.test_transmission.active: false fuel_distribution: techs: @@ -307,7 +307,7 @@ overrides: carrier_out: electricity nodes: a: - inherit: init_nodes + template: init_nodes techs: test_conversion: cost_flow_in: @@ -315,10 +315,10 @@ overrides: index: [[monetary, coal], [monetary, gas]] dims: [costs, carriers] b: - inherit: init_nodes + template: init_nodes - tech_groups.test_transmission.active: false - node_groups: + templates.test_transmission.active: false + templates: init_nodes: techs: test_demand_elec: @@ -357,7 +357,7 @@ overrides: flow_cap_per_unit: 10 storage_cap_per_unit: 15 - tech_groups.test_transmission.active: false + templates.test_transmission.active: false storage_purchase: nodes: @@ -372,7 +372,7 @@ overrides: index: monetary dims: costs - tech_groups.test_transmission.active: false + templates.test_transmission.active: false spores: config: @@ -419,7 +419,7 @@ overrides: config.build.operate_horizon: 12h investment_costs: - tech_groups: + templates: test_controller: lifetime: 25 cost_interest_rate: @@ -432,7 +432,7 @@ overrides: dims: costs var_costs: - tech_groups: + templates: test_controller: cost_flow_out: data: 0.1 diff --git a/tests/common/test_model/weighted_obj_func.yaml b/tests/common/test_model/weighted_obj_func.yaml index e2fb9e7a..39152136 100644 --- a/tests/common/test_model/weighted_obj_func.yaml +++ b/tests/common/test_model/weighted_obj_func.yaml @@ -71,7 +71,7 @@ techs: base_tech: demand sink_use_equals: file=demand_elec.csv -node_groups: +templates: node_techs: techs: cheap_polluting_supply: @@ -81,9 +81,9 @@ node_groups: nodes: a: - inherit: node_techs + template: node_techs b: - inherit: node_techs + template: node_techs overrides: illegal_string_cost_class: diff --git a/tests/common/util.py b/tests/common/util.py index eb75d2e7..c888a5f9 100644 --- a/tests/common/util.py +++ b/tests/common/util.py @@ -2,10 +2,11 @@ from pathlib import Path from typing import Literal +import xarray as xr + import calliope import calliope.backend import calliope.preprocess -import xarray as xr def build_test_model( diff --git a/tests/conftest.py b/tests/conftest.py index 6e7effa5..31f64a75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import numpy as np import pytest import xarray as xr + from calliope.attrdict import AttrDict from calliope.backend import latex_backend_model, pyomo_backend_model from calliope.preprocess import CalliopeMath @@ -53,6 +54,13 @@ def simple_supply(): return m +@pytest.fixture +def simple_supply_build_func(): + m = build_model({}, "simple_supply,two_hours,investment_costs") + m.build() + return m + + @pytest.fixture(scope="session") def supply_milp(): m = build_model({}, "supply_milp,two_hours,investment_costs") diff --git a/tests/test_backend_expression_parser.py b/tests/test_backend_expression_parser.py index e68f24f0..ded1feb9 100644 --- a/tests/test_backend_expression_parser.py +++ b/tests/test_backend_expression_parser.py @@ -6,6 +6,7 @@ import pyparsing as pp import pytest import xarray as xr + from calliope import exceptions from calliope.backend import expression_parser, helper_functions @@ -36,40 +37,40 @@ def as_array(self, x, y): return x * 10 + y -@pytest.fixture() +@pytest.fixture def valid_component_names(): return ["foo", "with_inf", "only_techs", "no_dims", "multi_dim_var", "no_dim_var"] -@pytest.fixture() +@pytest.fixture def base_parser_elements(): number, identifier = expression_parser.setup_base_parser_elements() return number, identifier -@pytest.fixture() +@pytest.fixture def number(base_parser_elements): return base_parser_elements[0] -@pytest.fixture() +@pytest.fixture def identifier(base_parser_elements): return base_parser_elements[1] -@pytest.fixture() +@pytest.fixture def evaluatable_identifier(identifier, valid_component_names): return expression_parser.evaluatable_identifier_parser( identifier, valid_component_names ) -@pytest.fixture() +@pytest.fixture def id_list(number, evaluatable_identifier): return expression_parser.list_parser(number, evaluatable_identifier) -@pytest.fixture() +@pytest.fixture def unsliced_param(): def _unsliced_param(valid_component_names): return expression_parser.unsliced_object_parser(valid_component_names) @@ -77,12 +78,12 @@ def _unsliced_param(valid_component_names): return _unsliced_param -@pytest.fixture() +@pytest.fixture def unsliced_param_with_obj_names(unsliced_param, valid_component_names): return unsliced_param(valid_component_names) -@pytest.fixture() +@pytest.fixture def sliced_param( number, identifier, evaluatable_identifier, unsliced_param_with_obj_names ): @@ -91,12 +92,12 @@ def sliced_param( ) -@pytest.fixture() +@pytest.fixture def sub_expression(identifier): return expression_parser.sub_expression_parser(identifier) -@pytest.fixture() +@pytest.fixture def helper_function( number, sliced_param, @@ -116,7 +117,7 @@ def helper_function( ) -@pytest.fixture() +@pytest.fixture def helper_function_no_nesting( number, sliced_param, @@ -157,7 +158,7 @@ def helper_function_one_parser_in_args(identifier, request): ) -@pytest.fixture() +@pytest.fixture def eval_kwargs(dummy_pyomo_backend_model): return { "helper_functions": helper_functions._registry["expression"], @@ -172,7 +173,7 @@ def eval_kwargs(dummy_pyomo_backend_model): } -@pytest.fixture() +@pytest.fixture def arithmetic( helper_function, number, sliced_param, sub_expression, unsliced_param_with_obj_names ): @@ -185,7 +186,7 @@ def arithmetic( ) -@pytest.fixture() +@pytest.fixture def helper_function_allow_arithmetic( number, sliced_param, @@ -209,22 +210,22 @@ def helper_function_allow_arithmetic( ) -@pytest.fixture() +@pytest.fixture def equation_comparison(arithmetic): return expression_parser.equation_comparison_parser(arithmetic) -@pytest.fixture() +@pytest.fixture def generate_equation(valid_component_names): return expression_parser.generate_equation_parser(valid_component_names) -@pytest.fixture() +@pytest.fixture def generate_slice(valid_component_names): return expression_parser.generate_slice_parser(valid_component_names) -@pytest.fixture() +@pytest.fixture def generate_sub_expression(valid_component_names): return expression_parser.generate_sub_expression_parser(valid_component_names) @@ -758,11 +759,11 @@ def var_left(self, request): def var_right(self, request): return request.param - @pytest.fixture() + @pytest.fixture def expected_left(self, var_left): return self.EXPR_PARAMS_AND_EXPECTED_EVAL[var_left] - @pytest.fixture() + @pytest.fixture def expected_right(self, var_right): return self.EXPR_PARAMS_AND_EXPECTED_EVAL[var_right] @@ -770,7 +771,7 @@ def expected_right(self, var_right): def operator(self, request): return request.param - @pytest.fixture() + @pytest.fixture def single_equation_simple(self, var_left, var_right, operator): return f"{var_left} {operator} {var_right}" @@ -849,7 +850,7 @@ def test_repr(self, equation_comparison): class TestAsMathString: - @pytest.fixture() + @pytest.fixture def latex_eval_kwargs(self, dummy_latex_backend_model): return { "helper_functions": helper_functions._registry["expression"], diff --git a/tests/test_backend_general.py b/tests/test_backend_general.py index 4a406007..f61443f5 100644 --- a/tests/test_backend_general.py +++ b/tests/test_backend_general.py @@ -1,11 +1,12 @@ import logging -import calliope import numpy as np import pandas as pd import pytest # noqa: F401 import xarray as xr +import calliope + from .common.util import build_test_model as build_model from .common.util import check_error_or_warning @@ -25,7 +26,7 @@ def built_model_cls_longnames(backend) -> calliope.Model: return m -@pytest.fixture() +@pytest.fixture def built_model_func_longnames(backend) -> calliope.Model: m = build_model({}, "simple_supply,two_hours,investment_costs") m.build(backend=backend, pre_validate_math_strings=False) @@ -33,7 +34,7 @@ def built_model_func_longnames(backend) -> calliope.Model: return m -@pytest.fixture() +@pytest.fixture def solved_model_func(backend) -> calliope.Model: m = build_model({}, "simple_supply,two_hours,investment_costs") m.build(backend=backend, pre_validate_math_strings=False) @@ -65,7 +66,7 @@ def solved_model_cls(backend) -> calliope.Model: return m -@pytest.fixture() +@pytest.fixture def built_model_func_updated_cost_flow_cap(backend, dummy_int: int) -> calliope.Model: m = build_model({}, "simple_supply,two_hours,investment_costs") m.build(backend=backend, pre_validate_math_strings=False) diff --git a/tests/test_backend_gurobi.py b/tests/test_backend_gurobi.py index ab934f2f..95821860 100755 --- a/tests/test_backend_gurobi.py +++ b/tests/test_backend_gurobi.py @@ -1,8 +1,9 @@ -import calliope.exceptions as exceptions import gurobipy import pytest # noqa: F401 import xarray as xr +import calliope.exceptions as exceptions + from .common.util import build_test_model as build_model from .common.util import check_error_or_warning @@ -24,7 +25,7 @@ def simple_supply_gurobi(self): m.solve() return m - @pytest.fixture() + @pytest.fixture def simple_supply_gurobi_func(self): m = build_model({}, "simple_supply,two_hours,investment_costs") m.build(backend="gurobi", pre_validate_math_strings=False) @@ -260,13 +261,13 @@ def test_to_lp_wrong_file_extension(self, simple_supply_gurobi, tmp_path): class TestShadowPrices: - @pytest.fixture() + @pytest.fixture def simple_supply(self): m = build_model({}, "simple_supply,two_hours,investment_costs") m.build(backend="gurobi") return m - @pytest.fixture() + @pytest.fixture def supply_milp(self): m = build_model({}, "supply_milp,two_hours,investment_costs") m.build(backend="gurobi") diff --git a/tests/test_backend_helper_functions.py b/tests/test_backend_helper_functions.py index 59541a06..9f696927 100644 --- a/tests/test_backend_helper_functions.py +++ b/tests/test_backend_helper_functions.py @@ -1,6 +1,7 @@ import numpy as np import pytest import xarray as xr + from calliope import exceptions from calliope.backend import helper_functions @@ -71,7 +72,7 @@ def parsing_kwargs(self, dummy_model_data): "return_type": "array", } - @pytest.fixture() + @pytest.fixture def is_defined_any(self, dummy_model_data): def _is_defined(drop_dims, dims): return ( @@ -82,7 +83,7 @@ def _is_defined(drop_dims, dims): return _is_defined - @pytest.fixture() + @pytest.fixture def is_defined_all(self, dummy_model_data): def _is_defined(drop_dims, dims): return ( diff --git a/tests/test_backend_latex_backend.py b/tests/test_backend_latex_backend.py index 559ac82f..446716c3 100644 --- a/tests/test_backend_latex_backend.py +++ b/tests/test_backend_latex_backend.py @@ -2,6 +2,7 @@ import pytest import xarray as xr + from calliope import exceptions from calliope.backend import latex_backend_model @@ -301,6 +302,11 @@ def test_create_obj_list(self, dummy_latex_backend_model): foobar + \textbf{Uses}: + \begin{itemize} + \item no_dims + \end{itemize} + \textbf{Default}: 0 \begin{equation} @@ -334,6 +340,10 @@ def test_create_obj_list(self, dummy_latex_backend_model): foobar + **Uses**: + + * no_dims + **Default**: 0 .. container:: scrolling-wrapper @@ -366,6 +376,10 @@ def test_create_obj_list(self, dummy_latex_backend_model): foobar + **Uses**: + + * [no_dims](#no_dims) + **Default**: 0 $$ @@ -436,7 +450,9 @@ def test_generate_math_doc_no_params(self, dummy_model_data, dummy_model_math): """ ) - def test_generate_math_doc_mkdocs_tabbed(self, dummy_model_data, dummy_model_math): + def test_generate_math_doc_mkdocs_features_tabs( + self, dummy_model_data, dummy_model_math + ): backend_model = latex_backend_model.LatexBackendModel( dummy_model_data, dummy_model_math ) @@ -448,7 +464,7 @@ def test_generate_math_doc_mkdocs_tabbed(self, dummy_model_data, dummy_model_mat "default": 0, }, ) - doc = backend_model.generate_math_doc(format="md", mkdocs_tabbed=True) + doc = backend_model.generate_math_doc(format="md", mkdocs_features=True) assert doc == textwrap.dedent( r""" @@ -477,18 +493,74 @@ def test_generate_math_doc_mkdocs_tabbed(self, dummy_model_data, dummy_model_mat """ ) - def test_generate_math_doc_mkdocs_tabbed_not_in_md( + def test_generate_math_doc_mkdocs_features_admonition( + self, dummy_model_data, dummy_model_math + ): + backend_model = latex_backend_model.LatexBackendModel( + dummy_model_data, dummy_model_math + ) + backend_model._add_all_inputs_as_parameters() + backend_model.add_global_expression( + "expr", + { + "equations": [{"expression": "no_dims + 1"}], + "description": "foobar", + "default": 0, + }, + ) + doc = backend_model.generate_math_doc(format="md", mkdocs_features=True) + assert doc == textwrap.dedent( + r""" + + ## Where + + ### expr + + foobar + + ??? info "Uses" + + * [no_dims](#no_dims) + + **Default**: 0 + + === "Math" + + $$ + \begin{array}{l} + \quad \textit{no\_dims} + 1\\ + \end{array} + $$ + + === "YAML" + + ```yaml + equations: + - expression: no_dims + 1 + ``` + + ## Parameters + + ### no_dims + + ??? info "Used in" + + * [expr](#expr) + """ + ) + + def test_generate_math_doc_mkdocs_features_not_in_md( self, dummy_model_data, dummy_model_math ): backend_model = latex_backend_model.LatexBackendModel( dummy_model_data, dummy_model_math ) with pytest.raises(exceptions.ModelError) as excinfo: - backend_model.generate_math_doc(format="rst", mkdocs_tabbed=True) + backend_model.generate_math_doc(format="rst", mkdocs_features=True) assert check_error_or_warning( excinfo, - "Cannot use MKDocs tabs when writing math to a non-Markdown file format.", + "Cannot use MKDocs features when writing math to a non-Markdown file format.", ) @pytest.mark.parametrize( diff --git a/tests/test_backend_module.py b/tests/test_backend_module.py index 7301a5a3..f220a3b9 100644 --- a/tests/test_backend_module.py +++ b/tests/test_backend_module.py @@ -1,6 +1,7 @@ """Test backend module functionality (`__init__.py`).""" import pytest + from calliope import backend from calliope.backend.backend_model import BackendModel from calliope.exceptions import BackendError diff --git a/tests/test_backend_parsing.py b/tests/test_backend_parsing.py index e54ca203..8847738c 100644 --- a/tests/test_backend_parsing.py +++ b/tests/test_backend_parsing.py @@ -2,11 +2,12 @@ from io import StringIO from unittest.mock import patch -import calliope import pyparsing as pp import pytest import ruamel.yaml as yaml import xarray as xr + +import calliope from calliope.backend import backend_model, expression_parser, parsing, where_parser from .common.util import check_error_or_warning @@ -19,7 +20,7 @@ def string_to_dict(yaml_string): return yaml_loader.load(StringIO(yaml_string)) -@pytest.fixture() +@pytest.fixture def component_obj(): setup_string = """ foreach: [A, A1] @@ -31,43 +32,43 @@ def component_obj(): return parsing.ParsedBackendComponent("constraints", "foo", variable_data) -@pytest.fixture() +@pytest.fixture def exists_array(component_obj, dummy_model_data): component_obj.sets = ["nodes", "techs"] return component_obj.combine_definition_matrix_and_foreach(dummy_model_data) -@pytest.fixture() +@pytest.fixture def valid_component_names(dummy_model_data): return ["foo", "bar", "baz", "foobar", *dummy_model_data.data_vars.keys()] -@pytest.fixture() +@pytest.fixture def expression_string_parser(valid_component_names): return expression_parser.generate_equation_parser(valid_component_names) -@pytest.fixture() +@pytest.fixture def arithmetic_string_parser(valid_component_names): return expression_parser.generate_arithmetic_parser(valid_component_names) -@pytest.fixture() +@pytest.fixture def slice_parser(valid_component_names): return expression_parser.generate_slice_parser(valid_component_names) -@pytest.fixture() +@pytest.fixture def sub_expression_parser(valid_component_names): return expression_parser.generate_sub_expression_parser(valid_component_names) -@pytest.fixture() +@pytest.fixture def where_string_parser(): return where_parser.generate_where_string_parser() -@pytest.fixture() +@pytest.fixture def expression_generator(): def _expression_generator(parse_string, where_string=None): expression_dict = {"expression": parse_string} @@ -78,7 +79,7 @@ def _expression_generator(parse_string, where_string=None): return _expression_generator -@pytest.fixture() +@pytest.fixture def generate_expression_list(component_obj, expression_string_parser): def _generate_expression_list(expression_list, **kwargs): return component_obj.generate_expression_list( @@ -99,7 +100,7 @@ def parse_sub_expressions_and_slices( } -@pytest.fixture() +@pytest.fixture def parsed_sub_expression_dict(component_obj, sub_expression_parser): def _parsed_sub_expression_dict(n_foo, n_bar): foos = ", ".join( @@ -122,7 +123,7 @@ def _parsed_sub_expression_dict(n_foo, n_bar): return _parsed_sub_expression_dict -@pytest.fixture() +@pytest.fixture def parsed_slice_dict(component_obj, slice_parser): def _parsed_slice_dict(n_tech1, n_tech2): techs1 = ", ".join(["{where: techs, expression: foo}" for i in range(n_tech1)]) @@ -141,7 +142,7 @@ def _parsed_slice_dict(n_tech1, n_tech2): return _parsed_slice_dict -@pytest.fixture() +@pytest.fixture def obj_with_sub_expressions_and_slices(): def _obj_with_sub_expressions_and_slices(equation_string): if isinstance(equation_string, str): @@ -181,7 +182,7 @@ def _obj_with_sub_expressions_and_slices(equation_string): return _obj_with_sub_expressions_and_slices -@pytest.fixture() +@pytest.fixture def equation_obj(expression_string_parser, where_string_parser): return parsing.ParsedBackendEquation( equation_name="foo", @@ -191,7 +192,7 @@ def equation_obj(expression_string_parser, where_string_parser): ) -@pytest.fixture() +@pytest.fixture def equation_sub_expression_obj(sub_expression_parser, where_string_parser): def _equation_sub_expression_obj(name): return parsing.ParsedBackendEquation( @@ -204,7 +205,7 @@ def _equation_sub_expression_obj(name): return _equation_sub_expression_obj -@pytest.fixture() +@pytest.fixture def equation_slice_obj(slice_parser, where_string_parser): def _equation_slice_obj(name): return parsing.ParsedBackendEquation( @@ -217,7 +218,7 @@ def _equation_slice_obj(name): return _equation_slice_obj -@pytest.fixture() +@pytest.fixture def dummy_backend_interface(dummy_model_data, dummy_model_math): # ignore the need to define the abstract methods from backend_model.BackendModel with patch.multiple(backend_model.BackendModel, __abstractmethods__=set()): @@ -239,7 +240,7 @@ def __init__(self): return DummyBackendModel() -@pytest.fixture() +@pytest.fixture def evaluatable_component_obj(valid_component_names): def _evaluatable_component_obj(equation_expressions): setup_string = f""" @@ -291,7 +292,7 @@ def evaluate_component_where( return component_obj, equation_where_aligned, request.param[1] -@pytest.fixture() +@pytest.fixture def evaluate_component_expression(evaluate_component_where, dummy_backend_interface): component_obj, equation_where, n_true = evaluate_component_where @@ -992,7 +993,7 @@ def test_evaluate_expression(self, evaluate_component_expression): class TestParsedConstraint: - @pytest.fixture() + @pytest.fixture def constraint_obj(self): dict_ = { "foreach": ["techs"], @@ -1044,7 +1045,7 @@ def test_parse_constraint_dict_evaluate_eq2( class TestParsedVariable: - @pytest.fixture() + @pytest.fixture def variable_obj(self): dict_ = {"foreach": ["techs"], "where": "False"} @@ -1066,7 +1067,7 @@ def test_parse_variable_dict_empty_eq1( class TestParsedObjective: - @pytest.fixture() + @pytest.fixture def objective_obj(self): dict_ = { "equations": [ diff --git a/tests/test_backend_pyomo.py b/tests/test_backend_pyomo.py index cc4d97de..b42cca63 100755 --- a/tests/test_backend_pyomo.py +++ b/tests/test_backend_pyomo.py @@ -1,18 +1,19 @@ import logging from itertools import product -import calliope -import calliope.backend -import calliope.exceptions as exceptions -import calliope.preprocess import numpy as np import pyomo.core as po import pyomo.kernel as pmo import pytest # noqa: F401 import xarray as xr -from calliope.backend import PyomoBackendModel from pyomo.core.kernel.piecewise_library.transforms import piecewise_sos2 +import calliope +import calliope.backend +import calliope.exceptions as exceptions +import calliope.preprocess +from calliope.backend import PyomoBackendModel + from .common.util import build_test_model as build_model from .common.util import check_error_or_warning, check_variable_exists @@ -1604,7 +1605,9 @@ def test_storage_initial_fractional_value(self): with pytest.raises(exceptions.ModelError) as error: m.build() - assert check_error_or_warning(error, "values larger than 1 are not allowed") + assert check_error_or_warning( + error, "requiring values within the interval [0, 1]" + ) class TestNewBackend: @@ -1618,7 +1621,7 @@ def simple_supply_updated_cost_flow_cap( simple_supply.backend.update_parameter("cost_flow_cap", dummy_int) return simple_supply - @pytest.fixture() + @pytest.fixture def temp_path(self, tmpdir_factory): return tmpdir_factory.mktemp("custom_math") @@ -2109,31 +2112,31 @@ def test_fails_on_not_reaching_bounds( class TestShadowPrices: - @pytest.fixture() + @pytest.fixture def simple_supply(self): m = build_model({}, "simple_supply,two_hours,investment_costs") m.build() return m - @pytest.fixture() + @pytest.fixture def supply_milp(self): m = build_model({}, "supply_milp,two_hours,investment_costs") m.build() return m - @pytest.fixture() + @pytest.fixture def simple_supply_with_yaml_shadow_prices(self): m = build_model({}, "simple_supply,two_hours,investment_costs,shadow_prices") m.build() return m - @pytest.fixture() + @pytest.fixture def simple_supply_yaml(self): m = build_model({}, "simple_supply,two_hours,investment_costs,shadow_prices") m.build() return m - @pytest.fixture() + @pytest.fixture def simple_supply_yaml_invalid(self): m = build_model( {}, @@ -2142,7 +2145,7 @@ def simple_supply_yaml_invalid(self): m.build() return m - @pytest.fixture() + @pytest.fixture def supply_milp_yaml(self): m = build_model({}, "supply_milp,two_hours,investment_costs,shadow_prices") m.build() @@ -2231,7 +2234,7 @@ def test_yaml_with_invalid_constraint(self, simple_supply_yaml_invalid): class TestValidateMathDict: LOGGER = "calliope.backend.backend_model" - @pytest.fixture() + @pytest.fixture def validate_math(self): def _validate_math(math_dict: dict): m = build_model({}, "simple_supply,investment_costs") diff --git a/tests/test_backend_pyomo_objective.py b/tests/test_backend_pyomo_objective.py index 2427daa1..2c9ffe58 100644 --- a/tests/test_backend_pyomo_objective.py +++ b/tests/test_backend_pyomo_objective.py @@ -1,7 +1,8 @@ -import calliope import pyomo.core as po import pytest +import calliope + from .common.util import build_test_model as build_model approx = pytest.approx diff --git a/tests/test_backend_where_parser.py b/tests/test_backend_where_parser.py index a64caead..69620155 100644 --- a/tests/test_backend_where_parser.py +++ b/tests/test_backend_where_parser.py @@ -2,6 +2,7 @@ import pyparsing import pytest import xarray as xr + from calliope.attrdict import AttrDict from calliope.backend import expression_parser, helper_functions, where_parser from calliope.exceptions import BackendError @@ -17,50 +18,50 @@ def parse_yaml(yaml_string): return AttrDict.from_yaml_string(yaml_string) -@pytest.fixture() +@pytest.fixture def base_parser_elements(): number, identifier = expression_parser.setup_base_parser_elements() return number, identifier -@pytest.fixture() +@pytest.fixture def number(base_parser_elements): return base_parser_elements[0] -@pytest.fixture() +@pytest.fixture def identifier(base_parser_elements): return base_parser_elements[1] -@pytest.fixture() +@pytest.fixture def data_var(identifier): return where_parser.data_var_parser(identifier) -@pytest.fixture() +@pytest.fixture def config_option(identifier): return where_parser.config_option_parser(identifier) -@pytest.fixture() +@pytest.fixture def bool_operand(): return where_parser.bool_parser() -@pytest.fixture() +@pytest.fixture def evaluatable_string(identifier): return where_parser.evaluatable_string_parser(identifier) -@pytest.fixture() +@pytest.fixture def helper_function(number, identifier, evaluatable_string): return expression_parser.helper_function_parser( evaluatable_string, number, generic_identifier=identifier ) -@pytest.fixture() +@pytest.fixture def comparison( evaluatable_string, number, helper_function, bool_operand, config_option, data_var ): @@ -74,19 +75,19 @@ def comparison( ) -@pytest.fixture() +@pytest.fixture def subset(identifier, evaluatable_string, number): return where_parser.subset_parser(identifier, evaluatable_string, number) -@pytest.fixture() +@pytest.fixture def where(bool_operand, helper_function, data_var, comparison, subset): return where_parser.where_parser( bool_operand, helper_function, data_var, comparison, subset ) -@pytest.fixture() +@pytest.fixture def eval_kwargs(dummy_pyomo_backend_model): return { "input_data": dummy_pyomo_backend_model.inputs, @@ -98,7 +99,7 @@ def eval_kwargs(dummy_pyomo_backend_model): } -@pytest.fixture() +@pytest.fixture def parse_where_string(eval_kwargs, where): def _parse_where_string(where_string): parsed_ = where.parse_string(where_string, parse_all=True) @@ -563,7 +564,7 @@ def test_where_malformed(self, where, instring): class TestAsMathString: - @pytest.fixture() + @pytest.fixture def latex_eval_kwargs(self, eval_kwargs, dummy_latex_backend_model): eval_kwargs["return_type"] = "math_string" eval_kwargs["backend_interface"] = dummy_latex_backend_model diff --git a/tests/test_cli.py b/tests/test_cli.py index 67cc97ee..7a3e3374 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,12 +2,13 @@ import tempfile from pathlib import Path -import calliope import importlib_resources import pytest # noqa: F401 -from calliope import AttrDict, cli from click.testing import CliRunner +import calliope +from calliope import AttrDict, cli + _MODEL_NATIONAL = ( importlib_resources.files("calliope") / "example_models" diff --git a/tests/test_constraint_results.py b/tests/test_constraint_results.py index 4b7afd19..06f3e633 100644 --- a/tests/test_constraint_results.py +++ b/tests/test_constraint_results.py @@ -1,6 +1,7 @@ -import calliope import pytest +import calliope + from .common.util import build_test_model as build_model approx = pytest.approx @@ -103,7 +104,7 @@ def _get_flow(model, flow): @pytest.mark.skip(reason="to be reimplemented by comparison to LP files") class TestModelSettings: - @pytest.fixture() + @pytest.fixture def run_model(self): def _run_model(feasibility, cap_val): override_dict = { @@ -131,15 +132,15 @@ def _run_model(feasibility, cap_val): return _run_model - @pytest.fixture() + @pytest.fixture def model_no_unmet(self, run_model): return run_model(True, 10) - @pytest.fixture() + @pytest.fixture def model_unmet_demand(self, run_model): return run_model(True, 5) - @pytest.fixture() + @pytest.fixture def model_unused_supply(self, run_model): return run_model(True, 15) @@ -191,7 +192,7 @@ def test_expected_infeasible_result(self, override, run_model): @pytest.mark.skip(reason="to be reimplemented by comparison to LP files") class TestEnergyCapacityPerStorageCapacity: - @pytest.fixture() + @pytest.fixture def model_file(self): return "flow_cap_per_storage_cap.yaml" diff --git a/tests/test_core_attrdict.py b/tests/test_core_attrdict.py index c4723197..c65ab18e 100644 --- a/tests/test_core_attrdict.py +++ b/tests/test_core_attrdict.py @@ -5,13 +5,14 @@ import numpy as np import pytest import ruamel.yaml as ruamel_yaml + from calliope.attrdict import _MISSING, AttrDict from .common.util import check_error_or_warning class TestAttrDict: - @pytest.fixture() + @pytest.fixture def regular_dict(self): d = { "a": 1, @@ -37,16 +38,16 @@ def regular_dict(self): d: """ - @pytest.fixture() + @pytest.fixture def yaml_filepath(self): this_path = Path(__file__).parent return this_path / "common" / "yaml_file.yaml" - @pytest.fixture() + @pytest.fixture def yaml_string(self): return self.setup_string - @pytest.fixture() + @pytest.fixture def attr_dict(self, regular_dict): d = regular_dict return AttrDict(d) diff --git a/tests/test_core_model.py b/tests/test_core_model.py index 4d8f91fb..e16ebfa4 100644 --- a/tests/test_core_model.py +++ b/tests/test_core_model.py @@ -1,11 +1,12 @@ import logging from contextlib import contextmanager +import pandas as pd +import pytest + import calliope import calliope.backend import calliope.preprocess -import pandas as pd -import pytest from .common.util import build_test_model as build_model from .common.util import check_error_or_warning diff --git a/tests/test_core_preprocess.py b/tests/test_core_preprocess.py index e0ec12c2..b449b73f 100644 --- a/tests/test_core_preprocess.py +++ b/tests/test_core_preprocess.py @@ -1,9 +1,10 @@ import warnings -import calliope -import calliope.exceptions as exceptions import pandas as pd import pytest + +import calliope +import calliope.exceptions as exceptions from calliope.attrdict import AttrDict from .common.util import build_test_model as build_model @@ -276,7 +277,7 @@ def test_model_version_mismatch(self): ) def test_unspecified_base_tech(self): - """All technologies and technology groups must specify a base_tech""" + """All technologies must specify a base_tech""" override = AttrDict.from_yaml_string( """ techs.test_supply_no_base_tech: @@ -292,7 +293,7 @@ def test_unspecified_base_tech(self): build_model(override_dict=override, scenario="simple_supply,one_day") def test_tech_as_base_tech(self): - """All technologies and technology groups must specify a base_tech""" + """All technologies must specify a base_tech""" override1 = AttrDict.from_yaml_string( """ techs.test_supply_tech_base_tech: diff --git a/tests/test_core_util.py b/tests/test_core_util.py index 09b66090..8e9175ba 100644 --- a/tests/test_core_util.py +++ b/tests/test_core_util.py @@ -3,12 +3,13 @@ import logging from pathlib import Path -import calliope import importlib_resources import jsonschema import numpy as np import pandas as pd import pytest + +import calliope from calliope.util import schema from calliope.util.generate_runs import generate_runs from calliope.util.logging import log_time @@ -180,7 +181,7 @@ def test_invalid_dict(self, to_validate, expected_path): ], ) - @pytest.fixture() + @pytest.fixture def base_math(self): return calliope.AttrDict.from_yaml( Path(calliope.__file__).parent / "math" / "plan.yaml" @@ -321,7 +322,7 @@ def sample_model_def_schema(self): """ return calliope.AttrDict.from_yaml_string(schema_string) - @pytest.fixture() + @pytest.fixture def expected_config_defaults(self): return pd.Series( { @@ -331,7 +332,7 @@ def expected_config_defaults(self): } ).sort_index() - @pytest.fixture() + @pytest.fixture def expected_model_def_defaults(self): return pd.Series( { diff --git a/tests/test_example_models.py b/tests/test_example_models.py index 9708e12e..6f19464e 100755 --- a/tests/test_example_models.py +++ b/tests/test_example_models.py @@ -1,10 +1,11 @@ import shutil from pathlib import Path -import calliope import numpy as np import pandas as pd import pytest + +import calliope from calliope import exceptions from .common.util import check_error_or_warning @@ -16,7 +17,7 @@ class TestModelPreproccessing: def test_preprocess_national_scale(self): calliope.examples.national_scale() - @pytest.mark.time_intensive() + @pytest.mark.time_intensive def test_preprocess_time_clustering(self): calliope.examples.time_clustering() @@ -112,7 +113,7 @@ def _example_tester(solver="cbc", solver_io=None): def test_nationalscale_example_results_cbc(self, example_tester): example_tester() - @pytest.mark.needs_gurobi_license() + @pytest.mark.needs_gurobi_license def test_nationalscale_example_results_gurobi(self, example_tester): pytest.importorskip("gurobipy") example_tester(solver="gurobi", solver_io="python") @@ -214,7 +215,7 @@ def example_tester(self, solver="cbc", solver_io=None): def test_nationalscale_example_results_cbc(self): self.example_tester() - @pytest.mark.needs_gurobi_license() + @pytest.mark.needs_gurobi_license @pytest.mark.filterwarnings( "ignore:(?s).*`gurobi_persistent`.*:calliope.exceptions.ModelWarning" ) @@ -230,7 +231,7 @@ def test_nationalscale_example_results_gurobi(self): assert np.allclose(gurobi_data.flow_cap, gurobi_persistent_data.flow_cap) assert np.allclose(gurobi_data.cost, gurobi_persistent_data.cost) - @pytest.fixture() + @pytest.fixture def base_model_data(self): model = calliope.examples.national_scale( time_subset=["2005-01-01", "2005-01-03"], scenario="spores" @@ -269,7 +270,7 @@ def test_fail_with_spores_as_input_dim(self, base_model_data): excinfo, "Cannot run SPORES with a SPORES dimension in any input" ) - @pytest.fixture() + @pytest.fixture def spores_with_override(self): def _spores_with_override(override_dict): result_without_override = self.example_tester() @@ -441,7 +442,7 @@ def example_tester(self, source_unit, solver="cbc", solver_io=None): def test_urban_example_results_area(self): self.example_tester("per_area") - @pytest.mark.needs_gurobi_license() + @pytest.mark.needs_gurobi_license def test_urban_example_results_area_gurobi(self): pytest.importorskip("gurobipy") self.example_tester("per_area", solver="gurobi", solver_io="python") @@ -449,7 +450,7 @@ def test_urban_example_results_area_gurobi(self): def test_urban_example_results_cap(self): self.example_tester("per_cap") - @pytest.mark.needs_gurobi_license() + @pytest.mark.needs_gurobi_license def test_urban_example_results_cap_gurobi(self): pytest.importorskip("gurobipy") self.example_tester("per_cap", solver="gurobi", solver_io="python") diff --git a/tests/test_io.py b/tests/test_io.py index 2d5b95b0..b496db6b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,10 +1,11 @@ import os import tempfile -import calliope -import calliope.io import pytest # noqa: F401 import xarray as xr + +import calliope +import calliope.io from calliope import exceptions from .common.util import check_error_or_warning diff --git a/tests/test_math.py b/tests/test_math.py index 6c64cdd7..c25aed51 100644 --- a/tests/test_math.py +++ b/tests/test_math.py @@ -4,9 +4,10 @@ import numpy as np import pytest -from calliope import AttrDict from pyomo.repn.tests import lp_diff +from calliope import AttrDict + from .common.util import build_lp, build_test_model CALLIOPE_DIR: Path = importlib.resources.files("calliope") @@ -148,6 +149,62 @@ def test_balance_storage(self, compare_lps): } compare_lps(model, custom_math, "balance_storage") + @pytest.mark.parametrize("with_export", [True, False]) + def test_cost_var_with_export(self, compare_lps, with_export): + """Test variable costs in the objective.""" + self.TEST_REGISTER.add("global_expressions.cost_var") + override = { + "techs.test_conversion_plus.cost_flow_out": { + "data": [1, 2], + "index": [["electricity", "monetary"], ["heat", "monetary"]], + "dims": ["carriers", "costs"], + }, + "techs.test_conversion_plus.cost_flow_in": { + "data": 4, + "index": "monetary", + "dims": "costs", + }, + } + if with_export: + override.update( + { + "techs.test_conversion_plus": { + "carrier_export": "heat", + "cost_export": { + "data": 3, + "index": [["heat", "monetary"]], + "dims": ["carriers", "costs"], + }, + }, + "techs.test_supply_elec": { + "carrier_export": "electricity", + "cost_export": { + "data": 5, + "index": "monetary", + "dims": "costs", + }, + }, + } + ) + model = build_test_model( + override, "conversion_and_conversion_plus,var_costs,two_hours" + ) + custom_math = { + # need the expression defined in a constraint/objective for it to appear in the LP file bounds + "objectives": { + "foo": { + "equations": [ + { + "expression": "sum(cost_var, over=[nodes, techs, costs, timesteps])" + } + ], + "sense": "minimise", + } + } + } + suffix = "_with_export" if with_export else "" + compare_lps(model, custom_math, f"cost_var{suffix}") + @pytest.mark.xfail(reason="not all base math is in the test config dict yet") def test_all_math_registered(self, base_math): """After running all the previous tests in the class, the base_math dict should be empty, i.e. all math has been tested""" @@ -177,7 +234,7 @@ def abs_filepath(self): def custom_math(self): return AttrDict.from_yaml(self.CUSTOM_MATH_DIR / self.YAML_FILEPATH) - @pytest.fixture() + @pytest.fixture def build_and_compare(self, abs_filepath, compare_lps): def _build_and_compare( filename: str, @@ -730,12 +787,12 @@ class TestNetImportShare(CustomMathExamples): "links_a_c_heat": { "from": "a", "to": "c", - "inherit": "test_transmission_heat", + "template": "test_transmission_heat", }, "links_a_c_elec": { "from": "a", "to": "c", - "inherit": "test_transmission_elec", + "template": "test_transmission_elec", }, }, } diff --git a/tests/test_preprocess_data_sources.py b/tests/test_preprocess_data_sources.py index af0817e9..818eb760 100644 --- a/tests/test_preprocess_data_sources.py +++ b/tests/test_preprocess_data_sources.py @@ -1,8 +1,9 @@ import logging -import calliope import pandas as pd import pytest + +import calliope from calliope.preprocess import data_sources from calliope.util.schema import CONFIG_SCHEMA, extract_from_schema diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py index b1dd044a..1cb32c0d 100644 --- a/tests/test_preprocess_model_data.py +++ b/tests/test_preprocess_model_data.py @@ -5,6 +5,7 @@ import pandas as pd import pytest import xarray as xr + from calliope import exceptions from calliope.attrdict import AttrDict from calliope.preprocess import data_sources, scenarios @@ -14,7 +15,7 @@ from .common.util import check_error_or_warning -@pytest.fixture() +@pytest.fixture def model_def(): model_def_path = Path(__file__).parent / "common" / "test_model" / "model.yaml" model_dict = AttrDict.from_yaml(model_def_path) @@ -24,7 +25,7 @@ def model_def(): return model_def_override, model_def_path -@pytest.fixture() +@pytest.fixture def data_source_list(model_def, init_config): model_def_dict, model_def_path = model_def return [ @@ -35,14 +36,14 @@ def data_source_list(model_def, init_config): ] -@pytest.fixture() +@pytest.fixture def init_config(config_defaults, model_def): model_def_dict, _ = model_def config_defaults.union(model_def_dict.pop("config"), allow_override=True) return config_defaults["init"] -@pytest.fixture() +@pytest.fixture def model_data_factory(model_def, init_config, model_defaults): model_def_dict, _ = model_def return ModelDataFactory( @@ -50,13 +51,13 @@ def model_data_factory(model_def, init_config, model_defaults): ) -@pytest.fixture() +@pytest.fixture def model_data_factory_w_params(model_data_factory: ModelDataFactory): model_data_factory.add_node_tech_data() return model_data_factory -@pytest.fixture() +@pytest.fixture def my_caplog(caplog): caplog.set_level(logging.DEBUG, logger="calliope.preprocess") return caplog @@ -442,7 +443,7 @@ def test_prepare_param_dict_not_lookup(self, model_data_factory: ModelDataFactor "foo | Cannot pass parameter data as a list unless the parameter is one of the pre-defined lookup arrays", ) - def test_inherit_defs_inactive( + def test_template_defs_inactive( self, my_caplog, model_data_factory: ModelDataFactory ): def_dict = {"A": {"active": False}} @@ -452,8 +453,11 @@ def test_inherit_defs_inactive( assert "(nodes, A) | Deactivated." in my_caplog.text assert not new_def_dict - def test_inherit_defs_nodes_inherit(self, model_data_factory: ModelDataFactory): - def_dict = {"A": {"inherit": "init_nodes", "my_param": 1}, "B": {"my_param": 2}} + def test_template_defs_nodes_inherit(self, model_data_factory: ModelDataFactory): + def_dict = { + "A": {"template": "init_nodes", "my_param": 1}, + "B": {"my_param": 2}, + } new_def_dict = model_data_factory._inherit_defs( dim_name="nodes", dim_dict=AttrDict(def_dict) ) @@ -467,11 +471,13 @@ def test_inherit_defs_nodes_inherit(self, model_data_factory: ModelDataFactory): "B": {"my_param": 2}, } - def test_inherit_defs_nodes_from_base(self, model_data_factory: ModelDataFactory): + def test_template_defs_nodes_from_base(self, model_data_factory: ModelDataFactory): + """Without a `dim_dict` to start off inheritance chaining, the `dim_name` will be used to find keys.""" new_def_dict = model_data_factory._inherit_defs(dim_name="nodes") assert set(new_def_dict.keys()) == {"a", "b", "c"} - def test_inherit_defs_techs(self, model_data_factory: ModelDataFactory): + def test_template_defs_techs(self, model_data_factory: ModelDataFactory): + """`dim_dict` overrides content of base model definition.""" model_data_factory.model_definition.set_key("techs.foo.base_tech", "supply") model_data_factory.model_definition.set_key("techs.foo.my_param", 2) @@ -481,9 +487,10 @@ def test_inherit_defs_techs(self, model_data_factory: ModelDataFactory): ) assert new_def_dict == {"foo": {"my_param": 1, "base_tech": "supply"}} - def test_inherit_defs_techs_inherit(self, model_data_factory: ModelDataFactory): + def test_template_defs_techs_inherit(self, model_data_factory: ModelDataFactory): + """Use of template is tracked in updated definition dictionary (as `techs_inheritance` here).""" model_data_factory.model_definition.set_key( - "techs.foo.inherit", "test_controller" + "techs.foo.template", "test_controller" ) model_data_factory.model_definition.set_key("techs.foo.base_tech", "supply") model_data_factory.model_definition.set_key("techs.foo.my_param", 2) @@ -500,7 +507,8 @@ def test_inherit_defs_techs_inherit(self, model_data_factory: ModelDataFactory): } } - def test_inherit_defs_techs_empty_def(self, model_data_factory: ModelDataFactory): + def test_template_defs_techs_empty_def(self, model_data_factory: ModelDataFactory): + """An empty `dim_dict` entry can be handled, by returning the model definition for that entry.""" model_data_factory.model_definition.set_key("techs.foo.base_tech", "supply") model_data_factory.model_definition.set_key("techs.foo.my_param", 2) @@ -510,9 +518,10 @@ def test_inherit_defs_techs_empty_def(self, model_data_factory: ModelDataFactory ) assert new_def_dict == {"foo": {"my_param": 2, "base_tech": "supply"}} - def test_inherit_defs_techs_missing_base_def( + def test_template_defs_techs_missing_base_def( self, model_data_factory: ModelDataFactory ): + """If inheriting from a template, checks against the schema will still be undertaken.""" def_dict = {"foo": {"base_tech": "supply"}} with pytest.raises(KeyError) as excinfo: model_data_factory._inherit_defs( @@ -528,56 +537,58 @@ def test_inherit_defs_techs_missing_base_def( [ ({"my_param": 1}, {"my_param": 1}, None), ( - {"inherit": "foo_group"}, - {"my_param": 1, "my_other_param": 2, "inherit": "foo_group"}, + {"template": "foo_group"}, + {"my_param": 1, "my_other_param": 2, "template": "foo_group"}, ["bar_group", "foo_group"], ), ( - {"inherit": "bar_group"}, - {"my_param": 2, "my_other_param": 2, "inherit": "bar_group"}, + {"template": "bar_group"}, + {"my_param": 2, "my_other_param": 2, "template": "bar_group"}, ["bar_group"], ), ( - {"inherit": "bar_group", "my_param": 3, "my_own_param": 1}, + {"template": "bar_group", "my_param": 3, "my_own_param": 1}, { "my_param": 3, "my_other_param": 2, "my_own_param": 1, - "inherit": "bar_group", + "template": "bar_group", }, ["bar_group"], ), ], ) - def test_climb_inheritance_tree( + def test_climb_template_tree( self, model_data_factory: ModelDataFactory, node_dict, expected_dict, expected_inheritance, ): + """Templates should be found and applied in order of 'ancestry' (newer dict keys replace older ones if they overlap).""" group_dict = { - "foo_group": {"inherit": "bar_group", "my_param": 1}, + "foo_group": {"template": "bar_group", "my_param": 1}, "bar_group": {"my_param": 2, "my_other_param": 2}, } - model_data_factory.model_definition["node_groups"] = AttrDict(group_dict) - new_dict, inheritance = model_data_factory._climb_inheritance_tree( + model_data_factory.model_definition["templates"] = AttrDict(group_dict) + new_dict, inheritance = model_data_factory._climb_template_tree( AttrDict(node_dict), "nodes", "A" ) assert new_dict == expected_dict assert inheritance == expected_inheritance - def test_climb_inheritance_tree_missing_ancestor( + def test_climb_template_tree_missing_ancestor( self, model_data_factory: ModelDataFactory ): + """Referencing a template that doesn't exist in `templates` raises an error.""" group_dict = { - "foo_group": {"inherit": "bar_group", "my_param": 1}, + "foo_group": {"template": "bar_group", "my_param": 1}, "bar_group": {"my_param": 2, "my_other_param": 2}, } - model_data_factory.model_definition["node_groups"] = AttrDict(group_dict) + model_data_factory.model_definition["templates"] = AttrDict(group_dict) with pytest.raises(KeyError) as excinfo: - model_data_factory._climb_inheritance_tree( - AttrDict({"inherit": "not_there"}), "nodes", "A" + model_data_factory._climb_template_tree( + AttrDict({"template": "not_there"}), "nodes", "A" ) assert check_error_or_warning(excinfo, "(nodes, A) | Cannot find `not_there`") @@ -845,7 +856,7 @@ def test_raise_error_on_transmission_tech_in_node( class TestTopLevelParams: - @pytest.fixture() + @pytest.fixture def run_and_test(self, model_data_factory_w_params): def _run_and_test(in_dict, out_dict, dims): model_data_factory_w_params.model_definition["parameters"] = { @@ -1045,7 +1056,7 @@ def test_node_tech_active_false(self, my_caplog): assert "(nodes, b), (techs, test_storage) | Deactivated" in my_caplog.text def test_link_active_false(self, my_caplog): - overrides = {"tech_groups.test_transmission.active": False} + overrides = {"templates.test_transmission.active": False} model = build_model(overrides, "simple_storage,two_hours,investment_costs") # Ensure what should be gone is gone diff --git a/tests/test_preprocess_time.py b/tests/test_preprocess_time.py index 2ee93c6f..0402c623 100644 --- a/tests/test_preprocess_time.py +++ b/tests/test_preprocess_time.py @@ -1,5 +1,6 @@ import pandas as pd import pytest # noqa: F401 + from calliope import AttrDict, exceptions from .common.util import build_test_model