Merge branch 'main' into update-top-level-config-params

.github/.codecov.yml

@@ -3,4 +3,4 @@ coverage:
     patch: off
 
 comment:
-  layout: "diff, flags, files"
+  layout: "diff, flags, files"

.github/workflows/commit-ci.yml

@@ -40,5 +40,16 @@ jobs:
     - name: Install jupyter kernel
       run: python -m ipykernel install --user --name calliope
 
-    - name: run tests
+    - name: run tests without coverage
+      if: github.ref != 'refs/heads/main'
       run: pytest
+
+    - name: run tests with coverage
+      if: github.ref == 'refs/heads/main'
+      run: pytest --cov
+
+    - name: upload coverage report to Codecov
+      if: github.ref == 'refs/heads/main'
+      uses: codecov/codecov-action@v3
+      env:
+        directory: "./reports/coverage/"

.github/workflows/pr-ci.yml

@@ -69,11 +69,17 @@ jobs:
     - name: Install jupyter kernel
       run: python -m ipykernel install --user --name calliope
 
-    - name: run tests
+    - name: run tests with coverage
+      if: matrix.os == 'ubuntu-latest' && matrix.py3version == '11'
+      run: pytest --cov
+
+    - name: run tests without coverage
+      if: matrix.os != 'ubuntu-latest' || matrix.py3version != '11'
       run: pytest
 
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3
       if: matrix.os == 'ubuntu-latest' && matrix.py3version == '11'
-      env:
+      with:
+        fail_ci_if_error: true
         directory: "./reports/coverage/"

README.md

@@ -1,7 +1,7 @@
 [![Chat on Gitter](https://img.shields.io/gitter/room/calliope-project/calliope.svg?style=flat-square)](https://app.gitter.im/#/room/#calliope-project_calliope:gitter.im)
-[![Main branch build status](https://img.shields.io/azure-devops/build/calliope-project/371cbbaa-fa6b-4efb-9b23-c4283a8e33eb/1?style=flat-square)](https://dev.azure.com/calliope-project/calliope/_build?definitionId=1)
+[![Main branch build status](https://github.com/calliope-project/calliope/actions/workflows/commit-ci.yml/badge.svg?branch=main)](https://github.com/calliope-project/calliope/actions/workflows/commit-ci.yml)
 [![Documentation build status](https://img.shields.io/readthedocs/calliope.svg?style=flat-square)](https://readthedocs.org/projects/calliope/builds/)
-[![Test coverage](https://img.shields.io/codecov/c/github/calliope-project/calliope?style=flat-square&token=b4fd170f0e7b43679a8bf649719e1cea)](https://codecov.io/gh/calliope-project/calliope)
+[![Test coverage](https://codecov.io/gh/calliope-project/calliope/graph/badge.svg?token=UM542yaYrh)](https://codecov.io/gh/calliope-project/calliope)
 [![PyPI version](https://img.shields.io/pypi/v/calliope.svg?style=flat-square)](https://pypi.python.org/pypi/calliope)
 [![Anaconda.org/conda-forge version](https://img.shields.io/conda/vn/conda-forge/calliope.svg?style=flat-square&label=conda)](https://anaconda.org/conda-forge/calliope)
 [![JOSS DOI](https://img.shields.io/badge/JOSS-10.21105/joss.00825-green.svg?style=flat-square)](https://doi.org/10.21105/joss.00825)

doc/user/advanced_features.rst

@@ -136,15 +136,15 @@ None of the ``tech_groups`` appear in model results, they are only used to group
 Removing techs, locations and links
 -----------------------------------
 
-By specifying :yaml:`exists: false` in the model configuration, which can be done for example through overrides, model components can be removed for debugging or scenario analysis.
+By specifying :yaml:`active: false` in the model configuration, which can be done for example through overrides, model components can be removed for debugging or scenario analysis.
 
 This works for:
 
-* Techs: :yaml:`techs.tech_name.exists: false`
-* Locations: :yaml:`locations.location_name.exists: false`
-* Links: :yaml:`links.location1,location2.exists: false`
-* Techs at a specific location:  :yaml:`locations.location_name.techs.tech_name.exists: false`
-* Transmission techs at a specific location: :yaml:`links.location1,location2.techs.transmission_tech.exists: false`
+* Techs: :yaml:`techs.tech_name.active: false`
+* Locations: :yaml:`locations.location_name.active: false`
+* Links: :yaml:`links.location1,location2.active: false`
+* Techs at a specific location:  :yaml:`locations.location_name.techs.tech_name.active: false`
+* Transmission techs at a specific location: :yaml:`links.location1,location2.techs.transmission_tech.active: false`
 
 .. _operational_mode:
 

pyproject.toml

@@ -5,8 +5,8 @@ minversion = "6.0"
 # `--strict-markers` - Raise error on unexpected pytest markers being used (add new markers to `markers` config)
 # `-nauto` - parallelise over as many threads as possible (uses pytest-xdist). If debugging (`--pdb`), this will default to one thread.
 # `--nbmake --nbmake-kernel=calliope` - test example notebooks using the "calliope" notebook kernel (uses nbmake)
-# `--cov --cov-report=xml --cov-config=pyproject.toml` - generate coverage report for tests (uses pytest-cov; call `--no-cov` in CLI to switch off; `--cov-config` include to avoid bug)
-addopts = "-rav --dist=loadscope --strict-markers -nauto --nbmake --nbmake-kernel=calliope --cov --cov-report=xml --cov-config=pyproject.toml"
+# `--cov-report=xml --cov-config=pyproject.toml` - coverage report config for when running in tests (uses pytest-cov; call `--cov` in CLI to switch coverage on; `--cov-config` include to avoid bug)
+addopts = "-rav --dist=loadscope --strict-markers -nauto --nbmake --nbmake-kernel=calliope --cov-report=xml --cov-config=pyproject.toml"
 # TODO: add testpath once notebooks are fixed: "doc/_static/notebooks"
 testpaths = ["tests"]
 

requirements/base.txt

@@ -6,7 +6,7 @@ jsonschema >= 4.17, < 4.19
 natsort >= 8, < 9
 netcdf4 >= 1.2, < 1.7
 numpy >= 1, < 2
-pandas >= 2, < 3
+pandas >= 2, < 2.1
 pyomo >= 6.5, < 7
 pyparsing >= 3.0, < 3.1
 ruamel.yaml >= 0.17, < 0.18

src/calliope/config/defaults.yaml

@@ -438,20 +438,21 @@ techs:
         area_use: 0 # name: Cost of area use ¦ unit: m\ :sup:`-2` ¦
         source_cap: 0 # name: Cost of source flow capacity ¦ unit: kW :sup:`-1` ¦
         storage_cap: 0 # name: Cost of storage capacity ¦ unit: kWh :sup:`-1` ¦
-    exists: true
+        depreciation_rate: 1 # name: Depreciation rate for annualisation of investment costs ¦ unit: fraction ¦
+    active: true
 
 nodes:
   default_node:
     transmission_node: false # true only if no techs are defined (including complete omission of the `techs` key). Automatically added during processing if not defined by user.
     coordinates: {} # The node's x-y coordinates for distance calculations and plotting: {lat: ..., lon: ...} or {x: ..., y: ...}
     available_area: .inf # This node's available land area (required if constraining technology deployment by area).
     techs: null # A list of technologies, optionally with node-specific settings overriding the technology's global settings.
-    exists: true
+    active: true
 
 links:
   default_node_from,default_node_to:
     techs:
       default_tech:
         distance: null # Used for per_distance constraints, but automatically inferred from coordinates of nodes in a link if not given directly
 
-    exists: true
+    active: true

src/calliope/preprocess/checks.py

@@ -323,7 +323,7 @@ def _check_tech_final(
     """
     if tech_id not in model_run.techs:
         model_warnings.append(
-            "Tech {} was removed by setting ``exists: False`` - not checking "
+            "Tech {} was removed by setting ``active: False`` - not checking "
             "the consistency of its constraints at node {}.".format(tech_id, loc_id)
         )
         return model_warnings, errors

src/calliope/preprocess/model_run.py

@@ -261,8 +261,8 @@ def process_techs(config_model):
     debug_comments = AttrDict()
 
     for tech_id, tech_config in config_model.techs.items():
-        # If a tech specifies ``exists: false``, we skip it entirely
-        if not tech_config.get("exists", True):
+        # If a tech specifies ``active: false``, we skip it entirely
+        if not tech_config.get("active", True):
             continue
 
         tech_result = AttrDict()

src/calliope/preprocess/nodes.py

@@ -66,15 +66,15 @@ def process_nodes(model_config, modelrun_techs):
     # Kill any nodes that the modeller does not want to exist
     ##
     for loc in list(nodes.keys()):
-        if not nodes[loc].get("exists", True):
+        if not nodes[loc].get("active", True):
             nodes.del_key(loc)
 
     ##
     # Process technologies
     ##
     techs_to_delete = []
     for tech_name in techs_in:
-        if not techs_in[tech_name].get("exists", True):
+        if not techs_in[tech_name].get("active", True):
             techs_to_delete.append(tech_name)
             continue
         # Get inheritance chain generated in process_techs()
@@ -165,9 +165,9 @@ def process_nodes(model_config, modelrun_techs):
             )
 
             # Now merge the tech settings into the node-specific
-            # tech dict -- but if a tech specifies ``exists: false``,
+            # tech dict -- but if a tech specifies ``active: false``,
             # we kill it at this node
-            if not tech_settings.get("exists", True):
+            if not tech_settings.get("active", True):
                 node_techs_to_delete.append("{}.techs.{}".format(loc_name, tech_name))
             else:
                 nodes[loc_name].techs[tech_name].union(
@@ -182,22 +182,22 @@ def process_nodes(model_config, modelrun_techs):
     for link in links_in:
         loc_from, loc_to = [i.strip() for i in link.split(",")]
         # Skip this link entirely if it has been told not to exist
-        if not links_in[link].get("exists", True):
+        if not links_in[link].get("active", True):
             continue
         # Also skip this link - and warn about it - if it links to a
         # now-inexistant (because removed) node
         if loc_from not in nodes.keys() or loc_to not in nodes.keys():
             warnings.append(
                 "Not building the link {},{} because one or both of its "
                 "nodes have been removed from the model by setting "
-                "``exists: false``".format(loc_from, loc_to)
+                "``active: false``".format(loc_from, loc_to)
             )
             continue
         processed_transmission_techs = AttrDict()
         for tech_name in links_in[link].techs:
             # Skip techs that have been told not to exist
             # for this particular link
-            if not links_in[link].get_key("techs.{}.exists".format(tech_name), True):
+            if not links_in[link].get_key("techs.{}.active".format(tech_name), True):
                 continue
             if tech_name not in processed_transmission_techs:
                 tech_settings = AttrDict()

tests/common/test_model/scenarios.yaml

@@ -72,7 +72,7 @@ overrides:
             costs.monetary: { purchase: 1, flow_cap: 0 }
           test_demand_elec:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   supply_purchase:
     nodes:
@@ -84,7 +84,7 @@ overrides:
             costs.monetary.flow_cap: 0
           test_demand_elec:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   supply_milp:
     nodes:
@@ -96,8 +96,8 @@ overrides:
               units_max: 1
               flow_cap_per_unit: 15
           test_demand_elec:
-      b.exists: false
-    links.a,b.exists: false
+      b.active: false
+    links.a,b.active: false
 
   supply_export:
     techs:
@@ -146,7 +146,7 @@ overrides:
           test_demand_elec:
           test_demand_heat:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   conversion_plus_milp:
     nodes:
@@ -162,7 +162,7 @@ overrides:
           test_demand_elec:
           test_demand_heat:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   conversion_plus_purchase:
     nodes:
@@ -178,7 +178,7 @@ overrides:
           test_demand_elec:
           test_demand_heat:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   simple_conversion_plus:
     nodes:
@@ -191,7 +191,7 @@ overrides:
           test_demand_elec:
           test_demand_heat:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   simple_storage:
     nodes:
@@ -226,7 +226,7 @@ overrides:
               storage_cap_per_unit: 15
           test_demand_elec:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   storage_purchase:
     nodes:
@@ -238,7 +238,7 @@ overrides:
             costs.monetary: { purchase: 1 }
           test_demand_elec:
 
-    links.a,b.exists: false
+    links.a,b.active: false
 
   clustering:
     config.init.time:

tests/test_constraint_results.py

@@ -109,7 +109,7 @@ def run_model(self):
         def _run_model(feasibility, cap_val):
             override_dict = {
                 "nodes.a.techs": {"test_supply_elec": {}, "test_demand_elec": {}},
-                "links.a,b.exists": False,
+                "links.a,b.active": False,
                 # pick a time subset where demand is uniformally -10 throughout
                 "config.init.subset_time": [
                     "2005-01-01 06:00:00",

tests/test_core_preprocess.py

@@ -742,7 +742,7 @@ def test_inexistent_group_constraint_empty_loc_tech(self):
             excinfo, "Constraint group `mygroup` will be completely ignored"
         )
 
-        assert m._model_run.group_constraints.mygroup.get("exists", True) is False
+        assert m._model_run.group_constraints.mygroup.get("active", True) is False
 
     @pytest.mark.filterwarnings(
         "ignore:(?s).*Not building the link a,b:calliope.exceptions.ModelWarning"

tests/test_core_time.py

@@ -1057,7 +1057,7 @@ def test_invalid_csv_columns(self):
                 "d.techs": {"test_supply_elec": None, "test_demand_elec": None},
             },
             "links": {
-                "a,b": {"exists": False},
+                "a,b": {"active": False},
                 "c,d.techs": {"test_transmission_elec": None},
             },
         }

tests/test_example_models.py

@@ -419,7 +419,7 @@ def model_runner(
             },
         }
         if storage is False:
-            override.update({"techs.battery.exists": False, "techs.csp.exists": False})
+            override.update({"techs.battery.active": False, "techs.csp.active": False})
         if storage_inter_cluster and backend_runner == "solve":
             override["config.init.custom_math"] = ["storage_inter_cluster"]
 

tests/test_model_manipulation.py

@@ -5,15 +5,15 @@
 from .common.util import check_error_or_warning
 
 
-class TestExistsFalse:
+class TestActiveFalse:
     """
     Test removal of techs, nodes, links, and transmission techs
-    with the ``exists: False`` configuration option.
+    with the ``active: False`` configuration option.
 
     """
 
-    def test_tech_exists_false(self):
-        overrides = {"techs.test_storage.exists": False}
+    def test_tech_active_false(self):
+        overrides = {"techs.test_storage.active": False}
         with pytest.warns(exceptions.ModelWarning) as excinfo:
             model = build_model(overrides, "simple_storage,two_hours,investment_costs")
 
@@ -23,11 +23,11 @@ def test_tech_exists_false(self):
         # Ensure warnings were raised
         assert check_error_or_warning(
             excinfo,
-            "Tech test_storage was removed by setting ``exists: False`` - not checking the consistency of its constraints at node a.",
+            "Tech test_storage was removed by setting ``active: False`` - not checking the consistency of its constraints at node a.",
         )
 
-    def test_node_exists_false(self):
-        overrides = {"nodes.b.exists": False}
+    def test_node_active_false(self):
+        overrides = {"nodes.b.active": False}
         with pytest.warns(exceptions.ModelWarning) as excinfo:
             model = build_model(overrides, "simple_storage,two_hours,investment_costs")
 
@@ -37,11 +37,11 @@ def test_node_exists_false(self):
         # Ensure warnings were raised
         assert check_error_or_warning(
             excinfo,
-            "Not building the link a,b because one or both of its nodes have been removed from the model by setting ``exists: false``",
+            "Not building the link a,b because one or both of its nodes have been removed from the model by setting ``active: false``",
         )
 
-    def test_node_tech_exists_false(self):
-        overrides = {"nodes.b.techs.test_storage.exists": False}
+    def test_node_tech_active_false(self):
+        overrides = {"nodes.b.techs.test_storage.active": False}
         model = build_model(overrides, "simple_storage,two_hours,investment_costs")
 
         # Ensure what should be gone is gone
@@ -51,15 +51,15 @@ def test_node_tech_exists_false(self):
             .item()
         )
 
-    def test_link_exists_false(self):
-        overrides = {"links.a,b.exists": False}
+    def test_link_active_false(self):
+        overrides = {"links.a,b.active": False}
         model = build_model(overrides, "simple_storage,two_hours,investment_costs")
 
         # Ensure what should be gone is gone
         assert not model._model_data.inheritance.str.endswith("transmission").any()
 
-    def test_link_tech_exists_false(self):
-        overrides = {"links.a,b.techs.test_transmission_elec.exists": False}
+    def test_link_tech_active_false(self):
+        overrides = {"links.a,b.techs.test_transmission_elec.active": False}
         model = build_model(overrides, "simple_storage,two_hours,investment_costs")
 
         # Ensure what should be gone is gone