Merge branch 'main' into feature/split-legend-on-vcat

Signed-off-by: Nicolas Höning <[email protected]>

documentation/changelog.rst

@@ -9,10 +9,10 @@ v0.24.0 | October XX, 2024
 
 New features
 -------------
-* The data chart on the asset page splits up its color-coded sensor legend when showing more than 7 sensors, becoming a legend per subplot [see `PR #1176 <https://github.com/FlexMeasures/flexmeasures/pull/1176>`_ and `PR #1193 <https://github.com/FlexMeasures/flexmeasures/pull/1193>`_]
+* The data chart on the asset page splits up its color-coded sensor legend when showing more than 7 sensors, becoming a legend per subplot [see `PR #1176 <https://github.com/FlexMeasures/flexmeasures/pull/1176>`_ and `PR #1193 <https://github.com/FlexMeasures/flexmeasures/pull/1193>`_
+* Speed up loading the users page, by making the pagination backend-based and adding support for that in the API [see `PR #1160 <https://github.com/FlexMeasures/flexmeasures/pull/1160>`]
 * X-axis labels in CLI plots show datetime values in a readable and informative format [see `PR #1172 <https://github.com/FlexMeasures/flexmeasures/pull/1172>`_]
 
-
 Infrastructure / Support
 ----------------------
 

flexmeasures/api/v3_0/accounts.py

@@ -200,13 +200,14 @@ def patch(self, account_data: dict, id: int, account: Account):
             "logo_url",
             "consultancy_account_id",
         ]
-
-        # Compile modified fields string
-        modified_fields_str = ", ".join(
-            field
+        modified_fields = {
+            field: getattr(account, field)
             for field in fields_to_check
             if account_data.get(field) != getattr(account, field)
-        )
+        }
+
+        # Compile modified fields string
+        modified_fields_str = ", ".join(modified_fields.keys())
 
         for k, v in account_data.items():
             setattr(account, k, v)

flexmeasures/api/v3_0/users.py

@@ -1,27 +1,33 @@
+from __future__ import annotations
 from flask_classful import FlaskView, route
 from marshmallow import fields
+import marshmallow.validate as validate
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy import select
+from sqlalchemy import and_, select, func
+from flask_sqlalchemy.pagination import SelectPagination
 from webargs.flaskparser import use_kwargs
 from flask_security import current_user, auth_required
 from flask_security.recoverable import send_reset_password_instructions
 from flask_json import as_json
-from werkzeug.exceptions import Forbidden, Unauthorized
+from werkzeug.exceptions import Forbidden
 from flexmeasures.auth.policy import check_access
 
 from flexmeasures.data.models.audit_log import AuditLog
 from flexmeasures.data.models.user import User as UserModel, Account
 from flexmeasures.api.common.schemas.users import AccountIdField, UserIdField
+from flexmeasures.api.common.schemas.generic_assets import SearchFilterField
+from flexmeasures.api.v3_0.assets import get_accessible_accounts
+from flexmeasures.data.queries.users import query_users_by_search_terms
+from flexmeasures.data.schemas.account import AccountSchema
 from flexmeasures.data.schemas.users import UserSchema
 from flexmeasures.data.services.users import (
-    get_users,
     set_random_password,
     remove_cookie_and_token_access,
     get_audit_log_records,
 )
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data import db
-from flexmeasures.utils.time_utils import server_now
+from flexmeasures.utils.time_utils import server_now, naturalized_datetime_str
 
 """
 API endpoints to manage users.
@@ -33,6 +39,7 @@
 user_schema = UserSchema()
 users_schema = UserSchema(many=True)
 partial_user_schema = UserSchema(partial=True)
+account_schema = AccountSchema()
 
 
 class UserAPI(FlaskView):
@@ -45,13 +52,27 @@ class UserAPI(FlaskView):
         {
             "account": AccountIdField(data_key="account_id", load_default=None),
             "include_inactive": fields.Bool(load_default=False),
+            "page": fields.Int(
+                required=False, validate=validate.Range(min=1), default=1
+            ),
+            "per_page": fields.Int(
+                required=False, validate=validate.Range(min=1), default=1
+            ),
+            "filter": SearchFilterField(required=False, default=None),
         },
         location="query",
     )
     @as_json
-    def index(self, account: Account, include_inactive: bool = False):
-        """API endpoint to list all users.
-
+    def index(
+        self,
+        account: Account,
+        include_inactive: bool = False,
+        page: int | None = None,
+        per_page: int | None = None,
+        filter: str | None = None,
+    ):
+        """
+        API endpoint to list all users.
 
         .. :quickref: User; Download user list
 
@@ -91,26 +112,63 @@ def index(self, account: Account, include_inactive: bool = False):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
-
         if account is not None:
             check_access(account, "read")
             accounts = [account]
         else:
-            accounts = []
-            for account in db.session.scalars(select(Account)).all():
-                try:
-                    check_access(account, "read")
-                    accounts.append(account)
-                except (Forbidden, Unauthorized):
-                    pass
-
-        users = []
-        for account in accounts:
-            users += get_users(
-                account_name=account.name,
-                only_active=not include_inactive,
+            accounts = get_accessible_accounts()
+
+        filter_statement = UserModel.account_id.in_([a.id for a in accounts])
+
+        if include_inactive is False:
+            filter_statement = and_(filter_statement, UserModel.active.is_(True))
+
+        query = query_users_by_search_terms(
+            search_terms=filter, filter_statement=filter_statement
+        )
+
+        if page is not None:
+            num_records = db.session.scalar(
+                select(func.count(UserModel.id)).where(filter_statement)
+            )
+            paginated_users: SelectPagination = db.paginate(
+                query, per_page=per_page, page=page
             )
-        return users_schema.dump(users), 200
+
+            users_response: list = [
+                {
+                    **user_schema.dump(user),
+                    "account": account_schema.dump(user.account),
+                    "flexmeasures_roles": [
+                        role.name for role in user.flexmeasures_roles
+                    ],
+                    "last_login_at": naturalized_datetime_str(user.last_login_at),
+                    "last_seen_at": naturalized_datetime_str(user.last_seen_at),
+                }
+                for user in paginated_users.items
+            ]
+            response: dict | list = {
+                "data": users_response,
+                "num-records": num_records,
+                "filtered-records": paginated_users.total,
+            }
+        else:
+            users = db.session.execute(query).scalars().all()
+
+            response = [
+                {
+                    **user_schema.dump(user),
+                    "account": account_schema.dump(user.account),
+                    "flexmeasures_roles": [
+                        role.name for role in user.flexmeasures_roles
+                    ],
+                    "last_login_at": naturalized_datetime_str(user.last_login_at),
+                    "last_seen_at": naturalized_datetime_str(user.last_seen_at),
+                }
+                for user in users
+            ]
+
+        return response, 200
 
     @route("/<id>")
     @use_kwargs({"user": UserIdField(data_key="id")}, location="path")

flexmeasures/data/queries/users.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from sqlalchemy import select, Select, or_, and_
+
+from flexmeasures.data.models.user import User as UserModel, Account
+
+
+def query_users_by_search_terms(
+    search_terms: list[str] | None,
+    filter_statement: bool = True,
+) -> Select:
+    select_statement = select(UserModel)
+    if search_terms is not None:
+        filter_statement = filter_statement & and_(
+            *(
+                or_(
+                    UserModel.email.ilike(f"%{term}%"),
+                    UserModel.username.ilike(f"%{term}%"),
+                    UserModel.account.has(Account.name.ilike(f"%{term}%")),
+                )
+                for term in search_terms
+            )
+        )
+        query = select_statement.where(filter_statement).order_by(UserModel.id)
+    else:
+        query = select_statement.where(filter_statement).order_by(UserModel.id)
+    return query

flexmeasures/data/services/users.py

@@ -35,43 +35,6 @@ def get_user(id: str) -> User:
     return user
 
 
-def get_users(
-    account_name: str | None = None,
-    role_name: str | None = None,
-    account_role_name: str | None = None,
-    only_active: bool = True,
-) -> list[User]:
-    """Return a list of User objects.
-    The role_name parameter allows to filter by role.
-    Set only_active to False if you also want non-active users.
-    """
-    user_query = select(User)
-
-    if account_name is not None:
-        account = db.session.execute(
-            select(Account).filter_by(name=account_name)
-        ).scalar_one_or_none()
-        if not account:
-            raise NotFound(f"There is no account named {account_name}!")
-        user_query = user_query.filter_by(account=account)
-
-    if only_active:
-        user_query = user_query.filter(User.active.is_(True))
-
-    if role_name is not None:
-        role = db.session.execute(
-            select(Role).filter_by(name=role_name)
-        ).scalar_one_or_none()
-        if role:
-            user_query = user_query.filter(User.flexmeasures_roles.contains(role))
-
-    users = db.session.scalars(user_query).all()
-    if account_role_name is not None:
-        users = [u for u in users if u.account.has_role(account_role_name)]
-
-    return users
-
-
 def find_user_by_email(user_email: str, keep_in_session: bool = True) -> User:
     user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role)
     user = user_datastore.find_user(email=user_email)

flexmeasures/ui/crud/accounts.py

@@ -7,7 +7,6 @@
 from flexmeasures.ui.crud.api_wrapper import InternalApi
 from flexmeasures.ui.utils.view_utils import render_flexmeasures_template
 from flexmeasures.ui.crud.assets import get_assets_by_account
-from flexmeasures.ui.crud.users import get_users_by_account
 from flexmeasures.data.models.user import Account
 from flexmeasures.data import db
 
@@ -58,14 +57,12 @@ def get(self, account_id: str):
                 account["consultancy_account_name"] = consultancy_account.name
         assets = get_assets_by_account(account_id)
         assets += get_assets_by_account(account_id=None)
-        users = get_users_by_account(account_id, include_inactive=include_inactive)
         accounts = get_accounts()
         return render_flexmeasures_template(
             "crud/account.html",
             account=account,
             accounts=accounts,
             assets=assets,
-            users=users,
             include_inactive=include_inactive,
         )
 

flexmeasures/ui/crud/users.py

@@ -74,34 +74,14 @@ def process_internal_api_response(
     return user_data
 
 
-def get_users_by_account(
-    account_id: int | str, include_inactive: bool = False
-) -> list[User]:
-    get_users_response = InternalApi().get(
-        url_for(
-            "UserAPI:index",
-            account_id=account_id,
-            include_inactive=include_inactive,
-        )
-    )
-    users = [
-        process_internal_api_response(user, make_obj=True)
-        for user in get_users_response.json()
-    ]
-    return users
-
-
 def get_all_users(include_inactive: bool = False) -> list[User]:
     get_users_response = InternalApi().get(
         url_for(
             "UserAPI:index",
             include_inactive=include_inactive,
         )
     )
-    users = [
-        process_internal_api_response(user, make_obj=True)
-        for user in get_users_response.json()
-    ]
+    users = [user for user in get_users_response.json()]
     return users
 
 
@@ -113,10 +93,8 @@ class UserCrudUI(FlaskView):
     def index(self):
         """/users"""
         include_inactive = request.args.get("include_inactive", "0") != "0"
-        users = get_all_users(include_inactive)
-
         return render_flexmeasures_template(
-            "crud/users.html", users=users, include_inactive=include_inactive
+            "crud/users.html", include_inactive=include_inactive
         )
 
     @login_required

flexmeasures/ui/templates/admin/logged_in_user.html

@@ -24,7 +24,7 @@
         <h2>User Overview</h2>
         <small>Logged-in user: {{ logged_in_user.username }}</small>
         <div class="table-responsive"> 
-        <table class="table table-striped table-responsive">
+        <table class="table table-striped">
           <tbody>
             <tr>
               <td>Email address</td>