655 Add User Roles and Basic User Data to Db

.github/workflows/run-tests.yml

@@ -4,6 +4,7 @@ on:
     branches: [main]
   # This is also a reusable workflow that can be called from other workflows
   workflow_call:
+  workflow_dispatch:
 jobs:
   test-api:
     runs-on: ubuntu-latest

api/README.md

@@ -78,10 +78,13 @@ To launch the tests, run `tox`.
 
 `tox` will run the tests for this (`api`) project using `pytest` in an isolated virtual environment that is distinct from the virtual environment that you created following the instructions from [Usage - Development](#usage---development). By default `tox` will invoke `pytest` in `debug` mode, which uses 3rd party API mocking. The test cases can optionally be run without mocking by testing the application in `release` mode, using `tox -e releasetest` or `pytest --mode=release`.
 
-### Alembic migrations
+### Database Management
 
-When changing the database, you can automatically generate an alembic migration. Simply change the model however you want in `database.py`, run `alembic revision --autogenerate -m <name_of_migration>` to generate a new migration, and then run `alembic upgrade head` to upgrade your database to the latest revision.
-In the case of someone else adding a revision to the database, simply pull their changes to your repo and run `alembic upgrade head` to upgrade your local database to the latest revision.
+Running the API requires an up-to-date version of the HomeUniteUs database. You can generate an empty database by running this command from the `api` directory with a virtual environment enabled.
+
+`alembic upgrade head`
+
+The current database version can be viewed using the `alembic current` command, or by inspecting the database `alembic_version` table. See the `model` [readme](./openapi_server/models/) for more information about the database project.
 
 ### Dependency Management
 

api/alembic/env.py

@@ -65,11 +65,16 @@ def run_migrations_online() -> None:
 
     """
     print("ONLINE")
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section, {}),
-        prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
-    )
+    # Check for an existing connection before creating a new engine.
+    # pytest-alembic will hook into alembic by creating a connection
+    # with the test engine configuration.
+    connectable = context.config.attributes.get("connection", None)
+    if connectable is None:
+        connectable = engine_from_config(
+            config.get_section(config.config_ini_section, {}),
+            prefix="sqlalchemy.",
+            poolclass=pool.NullPool,
+        )
 
     with connectable.connect() as connection:
         context.configure(

api/alembic/versions/e4c8bb426528_add_user_types.py

@@ -0,0 +1,72 @@
+"""Add user types
+
+Revision ID: e4c8bb426528
+Revises: ec8b1c17739a
+Create Date: 2024-03-10 21:47:13.942845
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import text
+from openapi_server.models.user_roles import UserRole
+
+# revision identifiers, used by Alembic.
+revision = 'e4c8bb426528'
+down_revision = 'ec8b1c17739a'
+branch_labels = None
+depends_on = None
+
+def upgrade() -> None:
+    '''
+    1. Add one table:
+        1. role - Store available application user roles
+    2. Prepopulate the role table with four role types: Admin, Host, Guest, Coordinator
+    3. Update the user table to add the first, middle, last name, and role_id columns.
+      * All existing users will be given the first, last name "UNKNOWN"
+      * Assign all existing users to the Guest role.
+    4. Drop the host table. 
+      * There is no way to map host users back to the user table. We would need a user id foreign 
+        key, or at least an email address.
+    '''
+    role_table = op.create_table('role',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name')
+    )
+    op.bulk_insert(role_table,
+                   [{'name': UserRole.ADMIN.value},
+                    {'name': UserRole.HOST.value},
+                    {'name': UserRole.GUEST.value},
+                    {'name': UserRole.COORDINATOR.value}])
+    op.create_index(op.f('ix_role_id'), 'role', ['id'])
+
+    conn = op.get_bind()
+    guest_role_id = conn.execute(text("SELECT id FROM role WHERE name = 'Guest'")).fetchone()[0]
+
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        # Each existing user will get the first and last names "Unknown" by default
+        # and they will be assigned to the "Guest" user role.
+        batch_op.add_column(sa.Column('firstName', sa.String(length=255), nullable=False, server_default='Unknown'))
+        batch_op.add_column(sa.Column('middleName', sa.String(length=255), nullable=True))
+        batch_op.add_column(sa.Column('lastName', sa.String(length=255), nullable=True))
+        batch_op.add_column(sa.Column('role_id', sa.Integer, nullable=False, server_default=str(guest_role_id)))
+        batch_op.create_foreign_key('fk_user_role_id', 'role', ['role_id'], ['id'])
+
+    op.drop_table('host')
+
+def downgrade() -> None:
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_constraint('fk_user_role_id', type_='foreignkey')
+        batch_op.drop_column('lastName')
+        batch_op.drop_column('middleName')
+        batch_op.drop_column('firstName')
+
+    op.drop_index(op.f('ix_role_id'), table_name='role')
+    op.drop_table('role')
+    op.create_table('host',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+        )
+    op.create_index(op.f('ix_host_id'), 'host', ['id'])

api/openapi_server/app.py

@@ -15,6 +15,7 @@
     IntegerConverter
 )
 
+import openapi_server.models
 from openapi_server.models.database import DataAccessLayer
 from openapi_server.exceptions import AuthError, handle_auth_error
 from openapi_server.configs.registry import HUUConfigRegistry, HUUConfig
@@ -142,6 +143,11 @@ def create_app(test_config: HUUConfig = None):
     flask_app = connexion_app.app
     flask_app.config.from_object(config)       
 
-    DataAccessLayer.db_init(flask_app.config["DATABASE_URL"])        
+    current_db_version = openapi_server.models.version()
+    DataAccessLayer.db_init(flask_app.config["DATABASE_URL"])
+    if DataAccessLayer.revision_id() != current_db_version:
+        raise Exception(f"Database is out of date. Expected {current_db_version} but " 
+                        f"found {DataAccessLayer.revision_id()}. "
+                        "Please run 'alembic upgrade head' to upgrade.")
 
     return connexion_app

api/openapi_server/configs/mock_aws.py

@@ -6,7 +6,7 @@
 
 from openapi_server.exceptions import AuthError
 from openapi_server.controllers.admin_controller import removeUser
-from openapi_server.controllers.auth_controller import signUpHost
+from openapi_server.controllers.auth_controller import signUpAdmin
 
 class AWSTemporaryUserpool():
     '''
@@ -125,9 +125,11 @@ def create_test_users(self):
                 pass
 
             try:
-                signUpHost({
+                signUpAdmin({
                     "email": email,
-                    "password": user["password"]
+                    "password": user["password"],
+                    "firstName": "testuser_firstname",
+                    "lastName": "testuser_lastname"
                 })
                 self._auto_signup_user(email)
                 self.app.logger.info(f"Created test user: {email}")

api/openapi_server/controllers/auth_controller.py

@@ -10,7 +10,9 @@
 )
 from openapi_server.exceptions import AuthError
 from openapi_server.models.database import DataAccessLayer, User
-from sqlalchemy.exc import IntegrityError
+from openapi_server.repositories.user_repo import UserRepository
+from openapi_server.models.user_roles import UserRole
+from openapi_server.models.schema import user_schema
 from sqlalchemy import select
 
 from botocore.exceptions import ClientError
@@ -59,19 +61,21 @@ def get_token_auth_header():
     token = parts[1]
     return token
 
-def sign_up(body: dict):
+def sign_up(body: dict, role: UserRole):
     secret_hash = current_app.calc_secret_hash(body['email'])
 
-    with DataAccessLayer.session() as session:
-        user = User(email=body['email'])
-        session.add(user)
-        try:
-            session.commit()
-        except IntegrityError:
-            session.rollback()
-            raise AuthError({
-                "message": "A user with this email already exists."
-            }, 422)
+    try:
+        with DataAccessLayer.session() as db_session:
+            user_repo = UserRepository(db_session)
+            user_repo.add_user(
+                email=body['email'],
+                role=role,
+                firstName=body['firstName'],
+                middleName=body.get('middleName', ''),
+                lastName=body.get('lastName', '')
+            )
+    except Exception as error:
+        raise AuthError({"message": str(error)}, 400)
 
     try:
         response = current_app.boto_client.sign_up(
@@ -107,13 +111,16 @@ def sign_up(body: dict):
         msg = f"The parameters you provided are incorrect: {error}"
         raise AuthError({"message": msg}, 500)
 
+def signUpAdmin(body: dict):
+    return sign_up(body, UserRole.ADMIN)
+
 def signUpHost(body: dict):
     """Signup a new Host"""
-    return sign_up(body)
+    return sign_up(body, UserRole.HOST)
 
 def signUpCoordinator(body: dict):  # noqa: E501
     """Signup a new Coordinator"""
-    return sign_up(body)
+    return sign_up(body, UserRole.COORDINATOR)
 
 def sign_in(body: dict):
     secret_hash = current_app.calc_secret_hash(body['email'])
@@ -140,22 +147,22 @@ def sign_in(body: dict):
     access_token = response['AuthenticationResult']['AccessToken']
     refresh_token = response['AuthenticationResult']['RefreshToken']
 
-    # retrieve user data
-    user_data = current_app.boto_client.get_user(AccessToken=access_token)
+    user_data = None
+    with DataAccessLayer.session() as db_session:
+        user_repo = UserRepository(db_session)
+        signed_in_user = user_repo.get_user(body['email'])
+        user_data = user_schema.dump(signed_in_user)
 
     # set refresh token cookie
     session['refresh_token'] = refresh_token
-    session['username'] = user_data['Username']
+    session['username'] = body['email']
 
     # return user data json
     return {
         'token': access_token,
-        'user': {
-            'email': body['email']
-        }
+        'user': user_data
     }
 
-
 def resend_confirmation_code():
     '''
     Resends the registration confirmation code to the specified user (identified by email).
@@ -380,10 +387,13 @@ def confirm_forgot_password():
     return response
 
 def user(token_info):
+    user_data = None
+    with DataAccessLayer.session() as db_session:
+        user_repo = UserRepository(db_session)
+        signed_in_user = user_repo.get_user(token_info["Username"])
+        user_data = user_schema.dump(signed_in_user)
     return {
-      "user": {
-          "email": token_info["Username"]
-      }
+      "user": user_data
     }
 
 def private(token_info):

api/openapi_server/controllers/host_controller.py

@@ -1,18 +1,12 @@
 from flask import Response
 
-from openapi_server.models.database import Host, DataAccessLayer
-from openapi_server.models.schema import host_schema, hosts_schema
-from sqlalchemy import select
-
-# create host in database
-def create_host(body: dict) -> Response:
-    with DataAccessLayer.session() as session:
-        new_host = Host(name = body["name"])
-        session.add(new_host)
-        session.commit()
-        return host_schema.dump(new_host), 201
+from openapi_server.models.database import DataAccessLayer
+from openapi_server.models.schema import users_schema
+from openapi_server.models.user_roles import UserRole
+from openapi_server.repositories.user_repo import UserRepository
 
 def get_hosts() -> Response:
     with DataAccessLayer.session() as session:
-        all_hosts = session.scalars(select(Host)).all()
-        return hosts_schema.dump(all_hosts), 200
+        user_repo = UserRepository(session)
+        all_users = user_repo.get_users_with_role(UserRole.HOST)
+        return users_schema.dump(all_users), 200

api/openapi_server/models/README.md

@@ -5,10 +5,6 @@ classDiagram
 class alembic_version{
    *VARCHAR<32> version_num NOT NULL
 }
-class host{
-   *INTEGER id NOT NULL
-   VARCHAR name NOT NULL
-}
 class housing_program{
    *INTEGER id NOT NULL
    VARCHAR program_name NOT NULL
@@ -18,13 +14,74 @@ class housing_program_service_provider{
    *INTEGER id NOT NULL
    VARCHAR provider_name NOT NULL
 }
+class role{
+   *INTEGER id NOT NULL
+   VARCHAR name NOT NULL
+}
 class user{
    *INTEGER id NOT NULL
    VARCHAR email NOT NULL
+   VARCHAR<255> firstName NOT NULL
+   VARCHAR<255> lastName NOT NULL
+   VARCHAR<255> middleName
+   INTEGER role_id NOT NULL
 }
 housing_program_service_provider "1" -- "0..n" housing_program
+role "1" -- "0..n" user
 ```
 
+Notes:
+
+* The `role` table is pre-populated with four roles by the `alembic` migration script:
+  * Guest, Host, Coordinator, Admin
+
+## How to update the Data Model
+
+The HomeUniteUs database is versioned, using `alembic`. Every change to the database model increments the database version number, and must be submitted with a database migration script that is capable of upgrading and downgrading the database.
+
+When changing the database, you can automatically generate an alembic migration. The autogenerated migration scripts can [fail to detect a number of changes](https://alembic.sqlalchemy.org/en/latest/autogenerate.html#what-does-autogenerate-detect-and-what-does-it-not-detect), so you should always review, test, and possibly edit the migration scripts before committing the update.
+
+The recommended steps for developing the database model are as follows:
+
+### 1. Backup your local database
+
+Errors in the `alembic` migration script can corrupt your local data. Take a copy of your database before applying new migration scripts.
+
+### 2. Update the python `Model` objects
+
+Implement your model changes by updating the `sqlalchemy` model objects within the `models` project. Most models are within `database.py`, but they may be placed elsewhere in the future.
+
+### 3. Autogenerate a migration script
+
+Run `alembic revision --autogenerate -m <name_of_migration>` to generate a new migration. The autogenerated scripts often times miss important migration changes, but they provide an excellent start.
+
+### 4. Test and manually edit migration script
+
+Our test suite includes `alembic` migration tests that cycle up and down through the entire database history to check for common migration errors. A great way to test your new migration script is to run:
+
+`pytest -k test_alembic_migration`
+
+Fix any errors. Once the script passes our common tests, please add additional tests to `test_alembic_migration` that capture the newly introduced model requirements.
+
+### 5. Apply the new script to your local db copy
+
+Apply the final migration script to a copy of your local database by:
+
+1. Name your local copy `api\homeuniteus.db`
+2. Upgrade the database `alembic upgrade head`
+
+You can verify your change was successful by viewing the database objects directly, and querying the database. If you are using `SQLite` then you can view the database objects using tools like [DB Browser](https://sqlitebrowser.org/).
+
+### 6. [Optional] Update & test the model schema
+
+If your model objects need to be (de)serialized (to)from json then add a schema object using `marshmallow_sqlalchemy.SQLAlchemyAutoSchema`. The schema object should contain all of the serialization related validation logic.
+
+Each submitted schema change should include test cases to verify the serialization rules and validation. See `tests/test_schema.py` for examples.
+
+### 7. Update the data model diagram
+
+Follow the steps in [Data Model Generation](#data-model-generation) to create a new mermaid diagram of the updated datamodel. Add the new model to the [Data Model section](#data-model).
+
 ## Data Model Generation
 
 Our data model is autogenerated using `eralchemy2`. The `eralchemy2` tool relies on `pygraphviz` and `graphviz` which are not easily installed using pip. The tool is easier to install on linux than windows, but instructions are provided for both operating systems.