Merge branch 'main' into 1126-use-new-most_recent-parameter-in-status…

…-page-to-speed-it-up

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0  # New version tags can be found here: https://github.com/pycqa/flake8/tags
+    rev: 7.1.1  # New version tags can be found here: https://github.com/pycqa/flake8/tags
     hooks:
     - id: flake8
       name: flake8 (code linting)
 -   repo: https://github.com/psf/black
-    rev: 22.10.0  # New version tags can be found here: https://github.com/psf/black/tags
+    rev: 24.8.0  # New version tags can be found here: https://github.com/psf/black/tags
     hooks:
     - id: black
       name: black (code formatting)

Makefile

@@ -43,6 +43,10 @@ install-for-dev:
 	make ensure-deps-folder
 	pip-sync requirements/${PYV}/app.txt requirements/${PYV}/dev.txt requirements/${PYV}/test.txt
 	make install-flexmeasures
+# Locally install HiGS on macOS
+	if [ "$(shell uname)" = "Darwin" ]; then \
+		make install-highs-macos; \
+	fi
 
 install-for-test:
 	make install-pip-tools

documentation/changelog.rst

@@ -9,12 +9,16 @@ v0.24.0 | October XX, 2024
 
 New features
 -------------
-* The asset beliefs chart can now be configured to not combine legends, but show them per plot [see `PR #1176 <https://github.com/FlexMeasures/flexmeasures/pull/1176>`_]
-
+* The data chart on the asset page splits up its color-coded sensor legend when showing more than 7 sensors, becoming a legend per subplot [see `PR #1176 <https://github.com/FlexMeasures/flexmeasures/pull/1176>`_ and `PR #1193 <https://github.com/FlexMeasures/flexmeasures/pull/1193>`_
+* Speed up loading the users page, by making the pagination backend-based and adding support for that in the API [see `PR #1160 <https://github.com/FlexMeasures/flexmeasures/pull/1160>`]
+* X-axis labels in CLI plots show datetime values in a readable and informative format [see `PR #1172 <https://github.com/FlexMeasures/flexmeasures/pull/1172>`_]
+* Speed up loading the accounts page,by making the pagination backend-based and adding support for that in the API [see `PR #1196 <https://github.com/FlexMeasures/flexmeasures/pull/1196>`_]
 
 Infrastructure / Support
 ----------------------
 
+* For MacOS developers, install HiGHS solver automatically [see `PR #1187 <https://github.com/FlexMeasures/flexmeasures/pull/1187>`_]
+
 Bugfixes
 -----------
 
@@ -30,7 +34,7 @@ New features
 * New chart type on sensor page: histogram [see `PR #1143 <https://github.com/FlexMeasures/flexmeasures/pull/1143>`_]
 * Add basic sensor info to sensor page [see `PR #1115 <https://github.com/FlexMeasures/flexmeasures/pull/1115>`_]
 * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors/<id>/stats` endpoint to get sensor statistics [see `PR #1116 <https://github.com/FlexMeasures/flexmeasures/pull/1116>`_]
-* Added support for adding custom titles to the graphs on the asset page. This works via an extension to the sensors_to_show format. [see `PR #1125 <https://github.com/FlexMeasures/flexmeasures/pull/1125>`_ and `PR #1177 <https://github.com/FlexMeasures/flexmeasures/pull/1177>`_]
+* Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 <https://github.com/FlexMeasures/flexmeasures/pull/1125>`_ and `PR #1177 <https://github.com/FlexMeasures/flexmeasures/pull/1177>`_]
 * Support zoom-in action on the asset and sensor charts [see `PR #1130 <https://github.com/FlexMeasures/flexmeasures/pull/1130>`_]
 * Speed up loading the assets page, by making the pagination backend-based and adding support for that in the API, and by enabling to query all accounts one can see in a single call (for admins and consultants) [see `PR #988 <https://github.com/FlexMeasures/flexmeasures/pull/988>`_]
 * Added Primary and Secondary colors to account for white-labelled UI themes [see `PR #1137 <https://github.com/FlexMeasures/flexmeasures/pull/1137>`_]

documentation/cli/change_log.rst

@@ -4,14 +4,17 @@
 FlexMeasures CLI Changelog
 **********************
 
-since v.0.22.0 | June 29, 2024
+since v0.24.0 | October XX, 2024
 =================================
 
+* ``flexmeasures show beliefs`` shows datetime values on x-axis labels.
 
-* Add ``--resolution`` option to ``flexmeasures show chart`` to produce charts in different time resolutions.
+since v0.22.0 | June 29, 2024
+=================================
 
+* Add ``--resolution`` option to ``flexmeasures show chart`` to produce charts in different time resolutions.
 
-since v.0.21.0 | April 16, 2024
+since v0.21.0 | April 16, 2024
 =================================
 
 * Include started, deferred and scheduled jobs in the overview printed by the CLI command ``flexmeasures jobs show-queues``.

flexmeasures/api/common/schemas/users.py

@@ -38,8 +38,8 @@ class UserIdField(fields.Integer):
     """
 
     def __init__(self, *args, **kwargs):
-        kwargs["load_default"] = (
-            lambda: current_user if not current_user.is_anonymous else None
+        kwargs["load_default"] = lambda: (
+            current_user if not current_user.is_anonymous else None
         )
         super().__init__(*args, **kwargs)
 

flexmeasures/api/v3_0/accounts.py

@@ -5,14 +5,22 @@
 from webargs.flaskparser import use_kwargs, use_args
 from flask_security import current_user, auth_required
 from flask_json import as_json
+from sqlalchemy import or_, select, func
+
+from marshmallow import fields
+import marshmallow.validate as validate
+from flask_sqlalchemy.pagination import SelectPagination
+
 
 from flexmeasures.auth.policy import user_has_admin_access
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data.models.audit_log import AuditLog
-from flexmeasures.data.models.user import Account
+from flexmeasures.data.models.user import Account, User
+from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.services.accounts import get_accounts, get_audit_log_records
 from flexmeasures.api.common.schemas.users import AccountIdField
 from flexmeasures.data.schemas.account import AccountSchema
+from flexmeasures.api.common.schemas.generic_assets import SearchFilterField
 from flexmeasures.utils.time_utils import server_now
 
 """
@@ -34,29 +42,63 @@ class AccountAPI(FlaskView):
     decorators = [auth_required()]
 
     @route("", methods=["GET"])
+    @use_kwargs(
+        {
+            "page": fields.Int(
+                required=False, validate=validate.Range(min=1), default=1
+            ),
+            "per_page": fields.Int(
+                required=False, validate=validate.Range(min=1), default=10
+            ),
+            "filter": SearchFilterField(required=False, default=None),
+        },
+        location="query",
+    )
     @as_json
-    def index(self):
+    def index(
+        self,
+        page: int | None = None,
+        per_page: int | None = None,
+        filter: list[str] | None = None,
+    ):
         """API endpoint to list all accounts accessible to the current user.
 
         .. :quickref: Account; Download account list
 
         This endpoint returns all accessible accounts.
         Accessible accounts are your own account and accounts you are a consultant for, or all accounts for admins.
 
+        The endpoint supports pagination of the asset list using the `page` and `per_page` query parameters.
+
+            - If the `page` parameter is not provided, all assets are returned, without pagination information. The result will be a list of assets.
+            - If a `page` parameter is provided, the response will be paginated, showing a specific number of assets per page as defined by `per_page` (default is 10).
+            - If a search 'filter' such as 'solar "ACME corp"' is provided, the response will filter out assets where each search term is either present in their name or account name.
+              The response schema for pagination is inspired by https://datatables.net/manual/server-side#Returned-data
+
         **Example response**
 
         An example of one account being returned:
 
         .. sourcecode:: json
 
-            [
+        {
+            "data" : [
                 {
                     'id': 1,
                     'name': 'Test Account'
                     'account_roles': [1, 3],
                     'consultancy_account_id': 2,
+                    'primary_color': '#1a3443'
+                    'secondary_color': '#f1a122'
+                    'logo_url': 'https://example.com/logo.png'
                 }
-            ]
+            ],
+            "num-records" : 1,
+            "filtered-records" : 1
+
+        }
+
+        If no pagination is requested, the response only consists of the list under the "data" key.
 
         :reqheader Authorization: The authentication token
         :reqheader Content-Type: application/json
@@ -76,7 +118,48 @@ def index(self):
                 else []
             )
 
-        return accounts_schema.dump(accounts), 200
+        query = db.session.query(Account).filter(
+            Account.id.in_([a.id for a in accounts])
+        )
+
+        if filter:
+            search_terms = filter[0].split(" ")
+            query = query.filter(
+                or_(*[Account.name.ilike(f"%{term}%") for term in search_terms])
+            )
+
+        if page:
+            select_pagination: SelectPagination = db.paginate(
+                query, per_page=per_page, page=page
+            )
+
+            accounts_reponse: list = []
+            for account in select_pagination.items:
+                user_count_query = select(func.count(User.id)).where(
+                    User.account_id == account.id
+                )
+                asset_count_query = select(func.count(GenericAsset.id)).where(
+                    GenericAsset.account_id == account.id
+                )
+                user_count = db.session.execute(user_count_query).scalar()
+                asset_count = db.session.execute(asset_count_query).scalar()
+                accounts_reponse.append(
+                    {
+                        **account_schema.dump(account),
+                        "user_count": user_count,
+                        "asset_count": asset_count,
+                    }
+                )
+
+            response = {
+                "data": accounts_reponse,
+                "num-records": select_pagination.total,
+                "filtered-records": select_pagination.total,
+            }
+        else:
+            response = accounts_schema.dump(query.all(), many=True)
+
+        return response, 200
 
     @route("/<id>", methods=["GET"])
     @use_kwargs({"account": AccountIdField(data_key="id")}, location="path")
@@ -200,13 +283,14 @@ def patch(self, account_data: dict, id: int, account: Account):
             "logo_url",
             "consultancy_account_id",
         ]
-
-        # Compile modified fields string
-        modified_fields_str = ", ".join(
-            field
+        modified_fields = {
+            field: getattr(account, field)
             for field in fields_to_check
             if account_data.get(field) != getattr(account, field)
-        )
+        }
+
+        # Compile modified fields string
+        modified_fields_str = ", ".join(modified_fields.keys())
 
         for k, v in account_data.items():
             setattr(account, k, v)

flexmeasures/api/v3_0/assets.py

@@ -1,5 +1,4 @@
 from __future__ import annotations
-
 import json
 
 from flask import current_app

flexmeasures/api/v3_0/tests/utils.py

@@ -69,9 +69,9 @@ def message_for_trigger_schedule(
         "duration": "PT24H",  # Will be extended in case of targets that would otherwise lie beyond the schedule's end
     }
     if unknown_prices:
-        message[
-            "start"
-        ] = "2040-01-01T00:00:00+01:00"  # We have no beliefs in our test database about 2040 prices
+        message["start"] = (
+            "2040-01-01T00:00:00+01:00"  # We have no beliefs in our test database about 2040 prices
+        )
 
     message["flex-model"] = {
         "soc-at-start": 12.1,  # in kWh, according to soc-unit