From e4794bca931985ba71403bc7d6b02b8214b1d572 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 23 Jan 2024 20:26:56 +0000 Subject: [PATCH 01/19] refactor(admin): assume database and superuser already exist --- bin/init.sh | 23 ---------- docs/configuration/environment-variables.md | 47 --------------------- terraform/app_service.tf | 4 -- 3 files changed, 74 deletions(-) diff --git a/bin/init.sh b/bin/init.sh index 8a11d863e..c2a283c2a 100755 --- a/bin/init.sh +++ b/bin/init.sh @@ -1,33 +1,10 @@ #!/usr/bin/env bash set -eux -# make the path to the database file from environment or default -DB_DIR="${DJANGO_DB_DIR:-.}" -DB_FILE="${DB_DIR}/django.db" -DB_RESET="${DJANGO_DB_RESET:-true}" - -# remove existing (old) database file -if [[ $DB_RESET = true && -f $DB_FILE ]]; then - # rename then delete the new file - # trying to avoid a file lock on the existing file - # after marking it for deletion - mv "${DB_FILE}" "${DB_FILE}.old" - rm "${DB_FILE}.old" -fi - # run database migrations python manage.py migrate -# create a superuser account for backend admin access -# check DJANGO_ADMIN = true, default to false if empty or unset - -if [[ ${DJANGO_ADMIN:-false} = true ]]; then - python manage.py createsuperuser --no-input -else - echo "superuser: Django not configured for Admin access" -fi - # generate language *.mo files for use by Django python manage.py compilemessages diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 6a80d8881..32701a1c0 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -78,17 +78,6 @@ writable by the Django process._ By default, the base project directory (i.e. the root of the repository). -### `DJANGO_DB_RESET` - -!!! warning "Deployment configuration" - - You may change this setting when deploying the app to a non-localhost domain - -Boolean: - -- `True` (default): deletes the existing database file and runs fresh Django migrations. -- `False`: Django uses the existing database file. - ### `DJANGO_DEBUG` !!! warning "Deployment configuration" @@ -144,42 +133,6 @@ By default the application sends logs to `stdout`. Django's primary secret, keep this safe! -### `DJANGO_SUPERUSER_EMAIL` - -!!! warning "Deployment configuration" - - You may change this setting when deploying the app to a non-localhost domain - -!!! danger "Required configuration" - - This setting is required when `DJANGO_ADMIN` is `true` - -The email address of the Django Admin superuser created during initialization. - -### `DJANGO_SUPERUSER_PASSWORD` - -!!! warning "Deployment configuration" - - You may change this setting when deploying the app to a non-localhost domain - -!!! danger "Required configuration" - - This setting is required when `DJANGO_ADMIN` is `true` - -The password of the Django Admin superuser created during initialization. - -### `DJANGO_SUPERUSER_USERNAME` - -!!! warning "Deployment configuration" - - You may change this setting when deploying the app to a non-localhost domain - -!!! danger "Required configuration" - - This setting is required when `DJANGO_ADMIN` is `true` - -The username of the Django Admin superuser created during initialization. - ### `DJANGO_TRUSTED_ORIGINS` !!! warning "Deployment configuration" diff --git a/terraform/app_service.tf b/terraform/app_service.tf index a59e7dd30..9bc66a5da 100644 --- a/terraform/app_service.tf +++ b/terraform/app_service.tf @@ -69,12 +69,8 @@ resource "azurerm_linux_web_app" "main" { "DJANGO_ADMIN" = "${local.secret_prefix}django-admin)", "DJANGO_ALLOWED_HOSTS" = "${local.secret_prefix}django-allowed-hosts)", "DJANGO_DB_DIR" = "${local.secret_prefix}django-db-dir)", - "DJANGO_DB_RESET" = "${local.secret_prefix}django-db-reset)", "DJANGO_DEBUG" = local.is_prod ? null : "${local.secret_prefix}django-debug)", "DJANGO_LOG_LEVEL" = "${local.secret_prefix}django-log-level)", - "DJANGO_SUPERUSER_EMAIL" = "${local.secret_prefix}django-superuser-email)", - "DJANGO_SUPERUSER_PASSWORD" = "${local.secret_prefix}django-superuser-password)", - "DJANGO_SUPERUSER_USERNAME" = "${local.secret_prefix}django-superuser-username)", "DJANGO_RECAPTCHA_SECRET_KEY" = local.is_dev ? null : "${local.secret_prefix}django-recaptcha-secret-key)", "DJANGO_RECAPTCHA_SITE_KEY" = local.is_dev ? null : "${local.secret_prefix}django-recaptcha-site-key)", From 4625f250f48eb9071d9b8fdc79e4dfc6cb2ee625 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 23 Jan 2024 20:50:43 +0000 Subject: [PATCH 02/19] refactor(admin): admin interface is always enabled --- benefits/core/admin.py | 74 ++++++++-------- benefits/settings.py | 97 +++++++++------------ benefits/urls.py | 13 +-- docs/configuration/README.md | 6 +- docs/configuration/environment-variables.md | 7 -- docs/getting-started/README.md | 3 +- terraform/app_service.tf | 1 - 7 files changed, 85 insertions(+), 116 deletions(-) diff --git a/benefits/core/admin.py b/benefits/core/admin.py index 68c045698..bd32e1369 100644 --- a/benefits/core/admin.py +++ b/benefits/core/admin.py @@ -2,43 +2,43 @@ The core application: Admin interface configuration. """ +import logging import requests from django.conf import settings - -if settings.ADMIN: - import logging - from django.contrib import admin - from . import models - - logger = logging.getLogger(__name__) - - for model in [ - models.EligibilityType, - models.EligibilityVerifier, - models.PaymentProcessor, - models.PemData, - models.TransitAgency, - ]: - logger.debug(f"Register {model.__name__}") - admin.site.register(model) - - def pre_login_user(user, request): - logger.debug(f"Running pre-login callback for user: {user.username}") - token = request.session.get("google_sso_access_token") - if token: - headers = { - "Authorization": f"Bearer {token}", - } - - # Request Google user info to get name and email - url = "https://www.googleapis.com/oauth2/v3/userinfo" - response = requests.get(url, headers=headers, timeout=settings.REQUESTS_TIMEOUT) - user_data = response.json() - logger.debug(f"Updating admin user data from Google for user with email: {user_data['email']}") - - user.first_name = user_data["given_name"] - user.last_name = user_data["family_name"] - user.username = user_data["email"] - user.email = user_data["email"] - user.save() +from django.contrib import admin +from . import models + +logger = logging.getLogger(__name__) + + +for model in [ + models.EligibilityType, + models.EligibilityVerifier, + models.PaymentProcessor, + models.PemData, + models.TransitAgency, +]: + logger.debug(f"Register {model.__name__}") + admin.site.register(model) + + +def pre_login_user(user, request): + logger.debug(f"Running pre-login callback for user: {user.username}") + token = request.session.get("google_sso_access_token") + if token: + headers = { + "Authorization": f"Bearer {token}", + } + + # Request Google user info to get name and email + url = "https://www.googleapis.com/oauth2/v3/userinfo" + response = requests.get(url, headers=headers, timeout=settings.REQUESTS_TIMEOUT) + user_data = response.json() + logger.debug(f"Updating admin user data from Google for user with email: {user_data['email']}") + + user.first_name = user_data["given_name"] + user.last_name = user_data["family_name"] + user.username = user_data["email"] + user.email = user_data["email"] + user.save() diff --git a/benefits/settings.py b/benefits/settings.py index 3323fb03c..9a0496cbe 100644 --- a/benefits/settings.py +++ b/benefits/settings.py @@ -45,39 +45,33 @@ def RUNTIME_ENVIRONMENT(): # Application definition INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", "django.contrib.messages", "django.contrib.sessions", "django.contrib.staticfiles", + "django_google_sso", "benefits.core", "benefits.enrollment", "benefits.eligibility", "benefits.oauth", ] -if ADMIN: - GOOGLE_SSO_CLIENT_ID = os.environ.get("GOOGLE_SSO_CLIENT_ID", "secret") - GOOGLE_SSO_PROJECT_ID = os.environ.get("GOOGLE_SSO_PROJECT_ID", "benefits-admin") - GOOGLE_SSO_CLIENT_SECRET = os.environ.get("GOOGLE_SSO_CLIENT_SECRET", "secret") - GOOGLE_SSO_ALLOWABLE_DOMAINS = _filter_empty(os.environ.get("GOOGLE_SSO_ALLOWABLE_DOMAINS", "compiler.la").split(",")) - GOOGLE_SSO_STAFF_LIST = _filter_empty(os.environ.get("GOOGLE_SSO_STAFF_LIST", "").split(",")) - GOOGLE_SSO_SUPERUSER_LIST = _filter_empty(os.environ.get("GOOGLE_SSO_SUPERUSER_LIST", "").split(",")) - GOOGLE_SSO_LOGO_URL = "/static/img/icon/google_sso_logo.svg" - GOOGLE_SSO_SAVE_ACCESS_TOKEN = True - GOOGLE_SSO_PRE_LOGIN_CALLBACK = "benefits.core.admin.pre_login_user" - GOOGLE_SSO_SCOPES = [ - "openid", - "https://www.googleapis.com/auth/userinfo.email", - "https://www.googleapis.com/auth/userinfo.profile", - ] - - INSTALLED_APPS.extend( - [ - "django.contrib.admin", - "django.contrib.auth", - "django.contrib.contenttypes", - "django_google_sso", # Add django_google_sso - ] - ) +GOOGLE_SSO_CLIENT_ID = os.environ.get("GOOGLE_SSO_CLIENT_ID", "secret") +GOOGLE_SSO_PROJECT_ID = os.environ.get("GOOGLE_SSO_PROJECT_ID", "benefits-admin") +GOOGLE_SSO_CLIENT_SECRET = os.environ.get("GOOGLE_SSO_CLIENT_SECRET", "secret") +GOOGLE_SSO_ALLOWABLE_DOMAINS = _filter_empty(os.environ.get("GOOGLE_SSO_ALLOWABLE_DOMAINS", "compiler.la").split(",")) +GOOGLE_SSO_STAFF_LIST = _filter_empty(os.environ.get("GOOGLE_SSO_STAFF_LIST", "").split(",")) +GOOGLE_SSO_SUPERUSER_LIST = _filter_empty(os.environ.get("GOOGLE_SSO_SUPERUSER_LIST", "").split(",")) +GOOGLE_SSO_LOGO_URL = "/static/img/icon/google_sso_logo.svg" +GOOGLE_SSO_SAVE_ACCESS_TOKEN = True +GOOGLE_SSO_PRE_LOGIN_CALLBACK = "benefits.core.admin.pre_login_user" +GOOGLE_SSO_SCOPES = [ + "openid", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile", +] MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", @@ -91,16 +85,10 @@ def RUNTIME_ENVIRONMENT(): "django.middleware.clickjacking.XFrameOptionsMiddleware", "csp.middleware.CSPMiddleware", "benefits.core.middleware.ChangedLanguageEvent", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", ] -if ADMIN: - MIDDLEWARE.extend( - [ - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - ] - ) - if DEBUG: MIDDLEWARE.append("benefits.core.middleware.DebugSession") @@ -162,13 +150,12 @@ def RUNTIME_ENVIRONMENT(): ] ) -if ADMIN: - template_ctx_processors.extend( - [ - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - ] - ) +template_ctx_processors.extend( + [ + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] +) TEMPLATES = [ { @@ -193,25 +180,21 @@ def RUNTIME_ENVIRONMENT(): # Password validation -AUTH_PASSWORD_VALIDATORS = [] +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] -if ADMIN: - AUTH_PASSWORD_VALIDATORS.extend( - [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", - }, - ] - ) # Internationalization diff --git a/benefits/urls.py b/benefits/urls.py index 39f12915f..0a5d658ec 100644 --- a/benefits/urls.py +++ b/benefits/urls.py @@ -8,6 +8,7 @@ import logging from django.conf import settings +from django.contrib import admin from django.http import HttpResponse from django.urls import include, path @@ -46,12 +47,6 @@ def test_secret(request): urlpatterns.append(path("testsecret/", test_secret)) - -if settings.ADMIN: - from django.contrib import admin - - logger.debug("Register admin urls") - urlpatterns.append(path("admin/", admin.site.urls)) - urlpatterns.append(path("google_sso/", include("django_google_sso.urls", namespace="django_google_sso"))) -else: - logger.debug("Skip url registrations for admin") +logger.debug("Register admin urls") +urlpatterns.append(path("admin/", admin.site.urls)) +urlpatterns.append(path("google_sso/", include("django_google_sso.urls", namespace="django_google_sso"))) diff --git a/docs/configuration/README.md b/docs/configuration/README.md index 9e0a29e96..ef12dcede 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -55,10 +55,10 @@ from django.config import settings # ... -if settings.ADMIN: - # do something when admin is enabled +if settings.DEBUG: + # do something when debug is enabled else: - # do something else when admin is disabled + # do something else when debug is disabled ``` Through the [Django model][django-model] framework, `benefits.core.models` instances are used to access the configuration data: diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 32701a1c0..7967ca615 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -48,13 +48,6 @@ If blank or an invalid key, analytics events aren't captured (though may still b ## Django -### `DJANGO_ADMIN` - -Boolean: - -- `True`: activates Django's built-in admin interface for content authoring. -- `False` (default): skips this activation. - ### `DJANGO_ALLOWED_HOSTS` !!! warning "Deployment configuration" diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md index 4ddea7ce1..ac2b5475c 100644 --- a/docs/getting-started/README.md +++ b/docs/getting-started/README.md @@ -56,8 +56,7 @@ docker compose up client After initialization, the client is running running on `http://localhost:8000` by default. -If `DJANGO_ADMIN=true`, the backend administrative interface can be accessed at the `/admin` route using the superuser account -you setup as part of initialization. +The backend administrative interface can be accessed at the `/admin` route using the superuser account you setup as part of initialization. By default, sample values are used to initialize Django. Alternatively you may: diff --git a/terraform/app_service.tf b/terraform/app_service.tf index 9bc66a5da..f50175f95 100644 --- a/terraform/app_service.tf +++ b/terraform/app_service.tf @@ -66,7 +66,6 @@ resource "azurerm_linux_web_app" "main" { "REQUESTS_READ_TIMEOUT" = "${local.secret_prefix}requests-read-timeout)", # Django settings - "DJANGO_ADMIN" = "${local.secret_prefix}django-admin)", "DJANGO_ALLOWED_HOSTS" = "${local.secret_prefix}django-allowed-hosts)", "DJANGO_DB_DIR" = "${local.secret_prefix}django-db-dir)", "DJANGO_DEBUG" = local.is_prod ? null : "${local.secret_prefix}django-debug)", From d9b2c28c01003e2c3979b19b1ef1af8091c90d57 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 23 Jan 2024 21:00:13 +0000 Subject: [PATCH 03/19] test(admin): update unit test assertion --- tests/pytest/core/test_admin.py | 8 ++++++++ tests/pytest/core/test_settings.py | 11 ----------- 2 files changed, 8 insertions(+), 11 deletions(-) create mode 100644 tests/pytest/core/test_admin.py delete mode 100644 tests/pytest/core/test_settings.py diff --git a/tests/pytest/core/test_admin.py b/tests/pytest/core/test_admin.py new file mode 100644 index 000000000..086d60b36 --- /dev/null +++ b/tests/pytest/core/test_admin.py @@ -0,0 +1,8 @@ +import pytest + + +@pytest.mark.django_db +def test_admin_registered(client): + response = client.get("/admin") + + assert response.status_code == 301 diff --git a/tests/pytest/core/test_settings.py b/tests/pytest/core/test_settings.py deleted file mode 100644 index 1359d43ae..000000000 --- a/tests/pytest/core/test_settings.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - -from django.conf import settings - - -@pytest.mark.django_db -def test_admin_not_registered(client): - response = client.get("/admin") - - assert settings.ADMIN is False - assert response.status_code == 404 From 2174ac81f080d186638e942333ce3afe2fd9042e Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 8 Feb 2024 22:06:45 +0000 Subject: [PATCH 04/19] feat: add script for resetting the database --- bin/reset_db.sh | 21 +++++++++++++++++++++ docs/configuration/environment-variables.md | 12 ++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 bin/reset_db.sh diff --git a/bin/reset_db.sh b/bin/reset_db.sh new file mode 100644 index 000000000..d408ae2cc --- /dev/null +++ b/bin/reset_db.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -eux + +# remove database file + +# construct the path to the database file from environment or default +DB_DIR="${DJANGO_DB_DIR:-.}" +DB_FILE="${DB_DIR}/django.db" + +# -f forces the delete (and avoids an error when the file doesn't exist) +rm -f "${DB_FILE}" + +# run database migrations + +python manage.py migrate + +# create a superuser account for backend admin access +# (set username, email, and password using environment variables +# DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD) + +python manage.py createsuperuser --no-input diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 7967ca615..305131cf3 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -126,6 +126,18 @@ By default the application sends logs to `stdout`. Django's primary secret, keep this safe! +### `DJANGO_SUPERUSER_EMAIL` + +The email address of the Django Admin superuser created when resetting the database. + +### `DJANGO_SUPERUSER_PASSWORD` + +The password of the Django Admin superuser created when resetting the database. + +### `DJANGO_SUPERUSER_USERNAME` + +The username of the Django Admin superuser created when resetting the database. + ### `DJANGO_TRUSTED_ORIGINS` !!! warning "Deployment configuration" From 0e0f37cf6887df3d44b603951d046bb0b9ffe600 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 16:54:49 +0000 Subject: [PATCH 05/19] refactor(settings): consolidate template processors remove duplicate django.contrib.messages.context_processors.messages --- benefits/settings.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/benefits/settings.py b/benefits/settings.py index 9a0496cbe..5ece74f1d 100644 --- a/benefits/settings.py +++ b/benefits/settings.py @@ -134,6 +134,7 @@ def RUNTIME_ENVIRONMENT(): template_ctx_processors = [ "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", "benefits.core.context_processors.agency", "benefits.core.context_processors.active_agencies", @@ -150,13 +151,6 @@ def RUNTIME_ENVIRONMENT(): ] ) -template_ctx_processors.extend( - [ - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - ] -) - TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", From 84956df1d0b7ef6c73fe972e54ecbd3ef51a6b21 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 17:38:56 +0000 Subject: [PATCH 06/19] test(admin): assert redirects to login page --- tests/pytest/core/test_admin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/pytest/core/test_admin.py b/tests/pytest/core/test_admin.py index 086d60b36..e02280f50 100644 --- a/tests/pytest/core/test_admin.py +++ b/tests/pytest/core/test_admin.py @@ -3,6 +3,10 @@ @pytest.mark.django_db def test_admin_registered(client): - response = client.get("/admin") + response = client.get("/admin", follow=True) - assert response.status_code == 301 + assert response.status_code == 200 + assert ("/admin/", 301) in response.redirect_chain + assert ("/admin/login/?next=/admin/", 302) in response.redirect_chain + assert response.request["PATH_INFO"] == "/admin/login/" + assert "google_sso/login.html" in response.template_name From eeffe2ec43b5af53ae2a4f4870731dbe99579be0 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 17:40:37 +0000 Subject: [PATCH 07/19] test(admin): pre_login_user success and failure --- benefits/core/admin.py | 10 +++++-- tests/pytest/core/test_admin.py | 52 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/benefits/core/admin.py b/benefits/core/admin.py index bd32e1369..cb468b629 100644 --- a/benefits/core/admin.py +++ b/benefits/core/admin.py @@ -12,6 +12,9 @@ logger = logging.getLogger(__name__) +GOOGLE_USER_INFO_URL = "https://www.googleapis.com/oauth2/v3/userinfo" + + for model in [ models.EligibilityType, models.EligibilityVerifier, @@ -32,13 +35,14 @@ def pre_login_user(user, request): } # Request Google user info to get name and email - url = "https://www.googleapis.com/oauth2/v3/userinfo" - response = requests.get(url, headers=headers, timeout=settings.REQUESTS_TIMEOUT) + response = requests.get(GOOGLE_USER_INFO_URL, headers=headers, timeout=settings.REQUESTS_TIMEOUT) user_data = response.json() - logger.debug(f"Updating admin user data from Google for user with email: {user_data['email']}") + logger.debug(f"Updating user data from Google for user with email: {user_data['email']}") user.first_name = user_data["given_name"] user.last_name = user_data["family_name"] user.username = user_data["email"] user.email = user_data["email"] user.save() + else: + logger.warning("google_sso_access_token not found in session.") diff --git a/tests/pytest/core/test_admin.py b/tests/pytest/core/test_admin.py index e02280f50..36bfa3434 100644 --- a/tests/pytest/core/test_admin.py +++ b/tests/pytest/core/test_admin.py @@ -1,4 +1,12 @@ import pytest +from django.contrib.auth.models import User +import benefits.core.admin +from benefits.core.admin import GOOGLE_USER_INFO_URL, pre_login_user + + +@pytest.fixture +def model_AdminUser(): + return User.objects.create(email="user@calitp.org", first_name="", last_name="", username="") @pytest.mark.django_db @@ -10,3 +18,47 @@ def test_admin_registered(client): assert ("/admin/login/?next=/admin/", 302) in response.redirect_chain assert response.request["PATH_INFO"] == "/admin/login/" assert "google_sso/login.html" in response.template_name + + +@pytest.mark.django_db +def test_pre_login_user(mocker, model_AdminUser): + assert model_AdminUser.email == "user@calitp.org" + assert model_AdminUser.first_name == "" + assert model_AdminUser.last_name == "" + assert model_AdminUser.username == "" + + response_from_google = { + "username": "admin@calitp.org", + "given_name": "Admin", + "family_name": "User", + "email": "admin@calitp.org", + } + + mocked_request = mocker.Mock() + mocked_response = mocker.Mock() + mocked_response.json.return_value = response_from_google + requests_spy = mocker.patch("benefits.core.admin.requests.get", return_value=mocked_response) + + pre_login_user(model_AdminUser, mocked_request) + + requests_spy.assert_called_once() + assert GOOGLE_USER_INFO_URL in requests_spy.call_args.args + assert model_AdminUser.email == response_from_google["email"] + assert model_AdminUser.first_name == response_from_google["given_name"] + assert model_AdminUser.last_name == response_from_google["family_name"] + assert model_AdminUser.username == response_from_google["username"] + + +@pytest.mark.django_db +def test_pre_login_user_no_session_token(mocker, model_AdminUser): + mocked_request = mocker.Mock() + mocked_request.session.get.return_value = None + logger_spy = mocker.spy(benefits.core.admin, "logger") + + pre_login_user(model_AdminUser, mocked_request) + + assert model_AdminUser.email == "user@calitp.org" + assert model_AdminUser.first_name == "" + assert model_AdminUser.last_name == "" + assert model_AdminUser.username == "" + logger_spy.warning.assert_called_once() From f394a3c5b01e1e915254b4642dbc104560b84957 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 17:42:52 +0000 Subject: [PATCH 08/19] chore(settings): remove unused variable --- benefits/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/benefits/settings.py b/benefits/settings.py index 5ece74f1d..0ce3ffdde 100644 --- a/benefits/settings.py +++ b/benefits/settings.py @@ -22,8 +22,6 @@ def _filter_empty(ls): # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get("DJANGO_DEBUG", "False").lower() == "true" -ADMIN = os.environ.get("DJANGO_ADMIN", "False").lower() == "true" - ALLOWED_HOSTS = _filter_empty(os.environ.get("DJANGO_ALLOWED_HOSTS", "localhost").split(",")) From 6f9ba826180eb300d2c5e927e160c615379474a6 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 23:32:02 +0000 Subject: [PATCH 09/19] chore(config): add sample SUPERUSER env vars these have to be present in the .env file to reset the DB locally --- .env.sample | 3 +++ bin/reset_db.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index d619baf55..c79424341 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,6 @@ +DJANGO_SUPERUSER_USERNAME=benefits-admin +DJANGO_SUPERUSER_EMAIL=benefits-admin@calitp.org +DJANGO_SUPERUSER_PASSWORD=superuser12345! testsecret=Hello from the local environment! auth_provider_client_id=benefits-oauth-client-id courtesy_card_verifier_api_auth_key=server-auth-token diff --git a/bin/reset_db.sh b/bin/reset_db.sh index d408ae2cc..7dabf7e48 100644 --- a/bin/reset_db.sh +++ b/bin/reset_db.sh @@ -15,7 +15,7 @@ rm -f "${DB_FILE}" python manage.py migrate # create a superuser account for backend admin access -# (set username, email, and password using environment variables -# DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD) +# set username, email, and password using environment variables +# DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD python manage.py createsuperuser --no-input From 409391e6818780fdbf86c39d79311a743a304d8d Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 23:33:24 +0000 Subject: [PATCH 10/19] feat(devcontainer): startup with the reset_db script reset_db.sh reuses the existing init.sh for other initialization --- .devcontainer/devcontainer.json | 2 +- bin/reset_db.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index db78bfbcb..7cb87f035 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "service": "dev", "runServices": ["dev", "docs", "server"], "workspaceFolder": "/home/calitp/app", - "postStartCommand": ["/bin/bash", "bin/init.sh"], + "postStartCommand": ["/bin/bash", "bin/reset_db.sh"], "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"], "customizations": { "vscode": { diff --git a/bin/reset_db.sh b/bin/reset_db.sh index 7dabf7e48..126500c3a 100644 --- a/bin/reset_db.sh +++ b/bin/reset_db.sh @@ -10,9 +10,9 @@ DB_FILE="${DB_DIR}/django.db" # -f forces the delete (and avoids an error when the file doesn't exist) rm -f "${DB_FILE}" -# run database migrations +# run database migrations and other initialization -python manage.py migrate +bin/init.sh # create a superuser account for backend admin access # set username, email, and password using environment variables From c2b074fe6d526b372720778d389530bdf8066fc3 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 23:52:15 +0000 Subject: [PATCH 11/19] refactor(migrations): remove data migration, use local fixtures dumped the existing (prior to this deletion) data using Django manage.py, excluding some model types that are defined and recreated by other migrations: python manage.py dumpdata \ --exclude auth.permission \ --exclude auth.user \ --exclude contenttypes.contenttype > fixtures.json then cleaned up the labels/names of our sample data for consistency updates db_reset.sh to load these fixtures after migrations are run updates the Cypress tests to use the new fixture location for sample data --- benefits/core/migrations/0002_data.py | 335 -------------- benefits/core/migrations/local_fixtures.json | 453 +++++++++++++++++++ benefits/core/migrations/sample_agency.json | 9 - bin/reset_db.sh | 4 + tests/cypress/fixtures/README.md | 2 +- tests/cypress/fixtures/transit-agencies.js | 10 +- 6 files changed, 466 insertions(+), 347 deletions(-) delete mode 100644 benefits/core/migrations/0002_data.py create mode 100644 benefits/core/migrations/local_fixtures.json delete mode 100644 benefits/core/migrations/sample_agency.json mode change 100644 => 100755 bin/reset_db.sh diff --git a/benefits/core/migrations/0002_data.py b/benefits/core/migrations/0002_data.py deleted file mode 100644 index a78b8e495..000000000 --- a/benefits/core/migrations/0002_data.py +++ /dev/null @@ -1,335 +0,0 @@ -"""Data migration which loads configuration data for Benefits. -""" - -import json -import os - -from django.db import migrations - - -def load_data(app, *args, **kwargs): - EligibilityType = app.get_model("core", "EligibilityType") - - mst_senior_type = EligibilityType.objects.create( - name="senior", label="Senior Discount (MST)", group_id=os.environ.get("MST_SENIOR_GROUP_ID", "group1") - ) - mst_veteran_type = EligibilityType.objects.create( - name="veteran", - label="Veteran Discount (MST)", - group_id=os.environ.get("MST_VETERAN_GROUP_ID", "group3"), - ) - mst_courtesy_card_type = EligibilityType.objects.create( - name="courtesy_card", - label="Courtesy Card Discount (MST)", - group_id=os.environ.get("MST_COURTESY_CARD_GROUP_ID", "group2"), - ) - sacrt_senior_type = EligibilityType.objects.create( - name="senior", label="Senior Discount (SacRT)", group_id=os.environ.get("SACRT_SENIOR_GROUP_ID", "group3") - ) - sbmtd_senior_type = EligibilityType.objects.create( - name="senior", label="Senior Discount (SBMTD)", group_id=os.environ.get("SBMTD_SENIOR_GROUP_ID", "group4") - ) - sbmtd_mobility_pass_type = EligibilityType.objects.create( - name="mobility_pass", - label="Mobility Pass Discount (SBMTD)", - group_id=os.environ.get("SBMTD_MOBILITY_PASS_GROUP_ID", "group5"), - ) - - PemData = app.get_model("core", "PemData") - - mst_server_public_key = PemData.objects.create( - label="Eligibility server public key", - remote_url=os.environ.get( - "MST_SERVER_PUBLIC_KEY_URL", "https://raw.githubusercontent.com/cal-itp/eligibility-server/main/keys/server.pub" - ), - ) - - sbmtd_server_public_key = PemData.objects.create( - label="Eligibility server public key", - remote_url=os.environ.get( - "SBMTD_SERVER_PUBLIC_KEY_URL", "https://raw.githubusercontent.com/cal-itp/eligibility-server/main/keys/server.pub" - ), - ) - - client_private_key = PemData.objects.create( - text_secret_name="client-private-key", - label="Benefits client private key", - ) - - client_public_key = PemData.objects.create( - text_secret_name="client-public-key", - label="Benefits client public key", - ) - - mst_payment_processor_client_cert = PemData.objects.create( - text_secret_name="mst-payment-processor-client-cert", - label="MST payment processor client certificate", - ) - - mst_payment_processor_client_cert_private_key = PemData.objects.create( - text_secret_name="mst-payment-processor-client-cert-private-key", - label="MST payment processor client certificate private key", - ) - - mst_payment_processor_client_cert_root_ca = PemData.objects.create( - text_secret_name="mst-payment-processor-client-cert-root-ca", - label="MST payment processor client certificate root CA", - ) - - sacrt_payment_processor_client_cert = PemData.objects.create( - text_secret_name="sacrt-payment-processor-client-cert", - label="SacRT payment processor client certificate", - ) - - sacrt_payment_processor_client_cert_private_key = PemData.objects.create( - text_secret_name="sacrt-payment-processor-client-cert-private-key", - label="SacRT payment processor client certificate private key", - ) - - sacrt_payment_processor_client_cert_root_ca = PemData.objects.create( - text_secret_name="sacrt-payment-processor-client-cert-root-ca", - label="SacRT payment processor client certificate root CA", - ) - - sbmtd_payment_processor_client_cert = PemData.objects.create( - text_secret_name="sbmtd-payment-processor-client-cert", - label="SBMTD payment processor client certificate", - ) - - sbmtd_payment_processor_client_cert_private_key = PemData.objects.create( - text_secret_name="sbmtd-payment-processor-client-cert-private-key", - label="SBMTD payment processor client certificate private key", - ) - - sbmtd_payment_processor_client_cert_root_ca = PemData.objects.create( - text_secret_name="sbmtd-payment-processor-client-cert-root-ca", - label="SBMTD payment processor client certificate root CA", - ) - - AuthProvider = app.get_model("core", "AuthProvider") - - senior_auth_provider = AuthProvider.objects.create( - sign_out_button_template="core/includes/button--sign-out--login-gov.html", - sign_out_link_template="core/includes/link--sign-out--login-gov.html", - client_name=os.environ.get("SENIOR_AUTH_PROVIDER_CLIENT_NAME", "senior-benefits-oauth-client-name"), - client_id_secret_name="auth-provider-client-id", - authority=os.environ.get("AUTH_PROVIDER_AUTHORITY", "https://example.com"), - scope=os.environ.get("SENIOR_AUTH_PROVIDER_SCOPE", "verify:senior"), - claim=os.environ.get("SENIOR_AUTH_PROVIDER_CLAIM", "senior"), - scheme=os.environ.get("SENIOR_AUTH_PROVIDER_SCHEME", "dev-cal-itp_benefits"), - ) - - veteran_auth_provider = AuthProvider.objects.create( - sign_out_button_template="core/includes/button--sign-out--login-gov.html", - sign_out_link_template="core/includes/link--sign-out--login-gov.html", - client_name=os.environ.get("VETERAN_AUTH_PROVIDER_CLIENT_NAME", "veteran-benefits-oauth-client-name"), - client_id_secret_name="auth-provider-client-id", - authority=os.environ.get("AUTH_PROVIDER_AUTHORITY", "https://example.com"), - scope=os.environ.get("VETERAN_AUTH_PROVIDER_SCOPE", "verify:veteran"), - claim=os.environ.get("VETERAN_AUTH_PROVIDER_CLAIM", "veteran"), - scheme=os.environ.get("VETERAN_AUTH_PROVIDER_SCHEME", "vagov"), - ) - - EligibilityVerifier = app.get_model("core", "EligibilityVerifier") - - mst_senior_verifier = EligibilityVerifier.objects.create( - name=os.environ.get("MST_SENIOR_VERIFIER_NAME", "OAuth claims via Login.gov (MST)"), - active=os.environ.get("MST_SENIOR_VERIFIER_ACTIVE", "True").lower() == "true", - eligibility_type=mst_senior_type, - auth_provider=senior_auth_provider, - selection_label_template="eligibility/includes/selection-label--senior.html", - start_template="eligibility/start--senior.html", - ) - - mst_veteran_verifier = EligibilityVerifier.objects.create( - name=os.environ.get("MST_VETERAN_VERIFIER_NAME", "VA.gov - Veteran (MST)"), - active=os.environ.get("MST_VETERAN_VERIFIER_ACTIVE", "True").lower() == "true", - eligibility_type=mst_veteran_type, - auth_provider=veteran_auth_provider, - selection_label_template="eligibility/includes/selection-label--veteran.html", - start_template="eligibility/start--veteran.html", - ) - - mst_courtesy_card_verifier = EligibilityVerifier.objects.create( - name=os.environ.get("COURTESY_CARD_VERIFIER_NAME", "Eligibility Server Verifier"), - active=os.environ.get("COURTESY_CARD_VERIFIER_ACTIVE", "True").lower() == "true", - api_url=os.environ.get("COURTESY_CARD_VERIFIER_API_URL", "http://server:8000/verify"), - api_auth_header=os.environ.get("COURTESY_CARD_VERIFIER_API_AUTH_HEADER", "X-Server-API-Key"), - api_auth_key_secret_name="courtesy-card-verifier-api-auth-key", - eligibility_type=mst_courtesy_card_type, - public_key=mst_server_public_key, - jwe_cek_enc=os.environ.get("COURTESY_CARD_VERIFIER_JWE_CEK_ENC", "A256CBC-HS512"), - jwe_encryption_alg=os.environ.get("COURTESY_CARD_VERIFIER_JWE_ENCRYPTION_ALG", "RSA-OAEP"), - jws_signing_alg=os.environ.get("COURTESY_CARD_VERIFIER_JWS_SIGNING_ALG", "RS256"), - auth_provider=None, - selection_label_template="eligibility/includes/selection-label--mst-courtesy-card.html", - start_template="eligibility/start--mst-courtesy-card.html", - form_class="benefits.eligibility.forms.MSTCourtesyCard", - ) - - sacrt_senior_verifier = EligibilityVerifier.objects.create( - name=os.environ.get("SACRT_SENIOR_VERIFIER_NAME", "OAuth claims via Login.gov (SacRT)"), - active=os.environ.get("SACRT_SENIOR_VERIFIER_ACTIVE", "False").lower() == "true", - eligibility_type=sacrt_senior_type, - auth_provider=senior_auth_provider, - selection_label_template="eligibility/includes/selection-label--senior.html", - start_template="eligibility/start--senior.html", - ) - - sbmtd_senior_verifier = EligibilityVerifier.objects.create( - name=os.environ.get("SBMTD_SENIOR_VERIFIER_NAME", "OAuth claims via Login.gov (SBMTD)"), - active=os.environ.get("SBMTD_SENIOR_VERIFIER_ACTIVE", "False").lower() == "true", - eligibility_type=sbmtd_senior_type, - auth_provider=senior_auth_provider, - selection_label_template="eligibility/includes/selection-label--senior.html", - start_template="eligibility/start--senior.html", - ) - - sbmtd_mobility_pass_verifier = EligibilityVerifier.objects.create( - name=os.environ.get("MOBILITY_PASS_VERIFIER_NAME", "Eligibility Server Verifier"), - active=os.environ.get("MOBILITY_PASS_VERIFIER_ACTIVE", "True").lower() == "true", - api_url=os.environ.get("MOBILITY_PASS_VERIFIER_API_URL", "http://server:8000/verify"), - api_auth_header=os.environ.get("MOBILITY_PASS_VERIFIER_API_AUTH_HEADER", "X-Server-API-Key"), - api_auth_key_secret_name="mobility-pass-verifier-api-auth-key", - eligibility_type=sbmtd_mobility_pass_type, - public_key=sbmtd_server_public_key, - jwe_cek_enc=os.environ.get("MOBILITY_PASS_VERIFIER_JWE_CEK_ENC", "A256CBC-HS512"), - jwe_encryption_alg=os.environ.get("MOBILITY_PASS_VERIFIER_JWE_ENCRYPTION_ALG", "RSA-OAEP"), - jws_signing_alg=os.environ.get("MOBILITY_PASS_VERIFIER_JWS_SIGNING_ALG", "RS256"), - auth_provider=None, - selection_label_template="eligibility/includes/selection-label--sbmtd-mobility-pass.html", - start_template="eligibility/start--sbmtd-mobility-pass.html", - form_class="benefits.eligibility.forms.SBMTDMobilityPass", - ) - - PaymentProcessor = app.get_model("core", "PaymentProcessor") - - mst_payment_processor = PaymentProcessor.objects.create( - name=os.environ.get("MST_PAYMENT_PROCESSOR_NAME", "Test Payment Processor"), - api_base_url=os.environ.get("MST_PAYMENT_PROCESSOR_API_BASE_URL", "http://server:8000"), - api_access_token_endpoint=os.environ.get("MST_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_ENDPOINT", "access-token"), - api_access_token_request_key=os.environ.get("MST_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_KEY", "request_access"), - api_access_token_request_val=os.environ.get("MST_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_VAL", "REQUEST_ACCESS"), - card_tokenize_url=os.environ.get("MST_PAYMENT_PROCESSOR_CARD_TOKENIZE_URL", "http://server:8000/static/tokenize.js"), - card_tokenize_func=os.environ.get("MST_PAYMENT_PROCESSOR_CARD_TOKENIZE_FUNC", "tokenize"), - card_tokenize_env=os.environ.get("MST_PAYMENT_PROCESSOR_CARD_TOKENIZE_ENV", "test"), - client_cert=mst_payment_processor_client_cert, - client_cert_private_key=mst_payment_processor_client_cert_private_key, - client_cert_root_ca=mst_payment_processor_client_cert_root_ca, - customer_endpoint="customer", - customers_endpoint="customers", - group_endpoint="group", - ) - - sacrt_payment_processor = PaymentProcessor.objects.create( - name=os.environ.get("SACRT_PAYMENT_PROCESSOR_NAME", "Test Payment Processor"), - api_base_url=os.environ.get("SACRT_PAYMENT_PROCESSOR_API_BASE_URL", "http://server:8000"), - api_access_token_endpoint=os.environ.get("SACRT_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_ENDPOINT", "access-token"), - api_access_token_request_key=os.environ.get("SACRT_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_KEY", "request_access"), - api_access_token_request_val=os.environ.get("SACRT_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_VAL", "REQUEST_ACCESS"), - card_tokenize_url=os.environ.get("SACRT_PAYMENT_PROCESSOR_CARD_TOKENIZE_URL", "http://server:8000/static/tokenize.js"), - card_tokenize_func=os.environ.get("SACRT_PAYMENT_PROCESSOR_CARD_TOKENIZE_FUNC", "tokenize"), - card_tokenize_env=os.environ.get("SACRT_PAYMENT_PROCESSOR_CARD_TOKENIZE_ENV", "test"), - client_cert=sacrt_payment_processor_client_cert, - client_cert_private_key=sacrt_payment_processor_client_cert_private_key, - client_cert_root_ca=sacrt_payment_processor_client_cert_root_ca, - customer_endpoint="customer", - customers_endpoint="customers", - group_endpoint="group", - ) - - sbmtd_payment_processor = PaymentProcessor.objects.create( - name=os.environ.get("SBMTD_PAYMENT_PROCESSOR_NAME", "Test Payment Processor"), - api_base_url=os.environ.get("SBMTD_PAYMENT_PROCESSOR_API_BASE_URL", "http://server:8000"), - api_access_token_endpoint=os.environ.get("SBMTD_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_ENDPOINT", "access-token"), - api_access_token_request_key=os.environ.get("SBMTD_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_KEY", "request_access"), - api_access_token_request_val=os.environ.get("SBMTD_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_VAL", "REQUEST_ACCESS"), - card_tokenize_url=os.environ.get("SBMTD_PAYMENT_PROCESSOR_CARD_TOKENIZE_URL", "http://server:8000/static/tokenize.js"), - card_tokenize_func=os.environ.get("SBMTD_PAYMENT_PROCESSOR_CARD_TOKENIZE_FUNC", "tokenize"), - card_tokenize_env=os.environ.get("SBMTD_PAYMENT_PROCESSOR_CARD_TOKENIZE_ENV", "test"), - client_cert=sbmtd_payment_processor_client_cert, - client_cert_private_key=sbmtd_payment_processor_client_cert_private_key, - client_cert_root_ca=sbmtd_payment_processor_client_cert_root_ca, - customer_endpoint="customer", - customers_endpoint="customers", - group_endpoint="group", - ) - - TransitAgency = app.get_model("core", "TransitAgency") - - # load the sample data from a JSON file so that it can be accessed by Cypress as well - sample_agency_data = os.path.join(os.path.dirname(__file__), "sample_agency.json") - with open(sample_agency_data) as f: - sample_agency = json.load(f) - - mst_agency = TransitAgency.objects.create( - slug=sample_agency["slug"], - short_name=os.environ.get("MST_AGENCY_SHORT_NAME", sample_agency["short_name"]), - long_name=os.environ.get("MST_AGENCY_LONG_NAME", sample_agency["long_name"]), - agency_id=sample_agency["agency_id"], - merchant_id=sample_agency["merchant_id"], - info_url=sample_agency["info_url"], - phone=sample_agency["phone"], - active=True, - private_key=client_private_key, - public_key=client_public_key, - jws_signing_alg=os.environ.get("MST_AGENCY_JWS_SIGNING_ALG", "RS256"), - payment_processor=mst_payment_processor, - index_template="core/index--mst.html", - eligibility_index_template="eligibility/index--mst.html", - enrollment_success_template="enrollment/success--mst.html", - help_template="core/includes/help--mst.html", - ) - mst_agency.eligibility_types.set([mst_senior_type, mst_veteran_type, mst_courtesy_card_type]) - mst_agency.eligibility_verifiers.set([mst_senior_verifier, mst_veteran_verifier, mst_courtesy_card_verifier]) - - sacrt_agency = TransitAgency.objects.create( - slug="sacrt", - short_name=os.environ.get("SACRT_AGENCY_SHORT_NAME", "SacRT (sample)"), - long_name=os.environ.get("SACRT_AGENCY_LONG_NAME", "Sacramento Regional Transit (sample)"), - agency_id="sacrt", - merchant_id=os.environ.get("SACRT_AGENCY_MERCHANT_ID", "sacrt"), - info_url="https://sacrt.com/", - phone="916-321-2877", - active=os.environ.get("SACRT_AGENCY_ACTIVE", "True").lower() == "true", - private_key=client_private_key, - public_key=client_public_key, - jws_signing_alg=os.environ.get("SACRT_AGENCY_JWS_SIGNING_ALG", "RS256"), - payment_processor=sacrt_payment_processor, - index_template="core/index--sacrt.html", - eligibility_index_template="eligibility/index--sacrt.html", - enrollment_success_template="enrollment/success--sacrt.html", - ) - sacrt_agency.eligibility_types.set([sacrt_senior_type]) - sacrt_agency.eligibility_verifiers.set([sacrt_senior_verifier]) - - sbmtd_agency = TransitAgency.objects.create( - slug="sbmtd", - short_name=os.environ.get("SBMTD_AGENCY_SHORT_NAME", "SBMTD (sample)"), - long_name=os.environ.get("SBMTD_AGENCY_LONG_NAME", "Santa Barbara MTD (sample)"), - agency_id="sbmtd", - merchant_id=os.environ.get("SBMTD_AGENCY_MERCHANT_ID", "sbmtd"), - info_url="https://sbmtd.gov/taptoride/", - phone="805-963-3366", - active=os.environ.get("SBMTD_AGENCY_ACTIVE", "True").lower() == "true", - private_key=client_private_key, - public_key=client_public_key, - jws_signing_alg=os.environ.get("SBMTD_AGENCY_JWS_SIGNING_ALG", "RS256"), - payment_processor=sbmtd_payment_processor, - index_template="core/index--sbmtd.html", - eligibility_index_template="eligibility/index--sbmtd.html", - enrollment_success_template="enrollment/success--sbmtd.html", - help_template="core/includes/help--sbmtd.html", - ) - sbmtd_agency.eligibility_types.set([sbmtd_senior_type, sbmtd_mobility_pass_type]) - sbmtd_agency.eligibility_verifiers.set([sbmtd_senior_verifier, sbmtd_mobility_pass_verifier]) - - -class Migration(migrations.Migration): - dependencies = [ - ("core", "0001_initial"), - ] - - operations = [ - migrations.RunPython(load_data), - ] diff --git a/benefits/core/migrations/local_fixtures.json b/benefits/core/migrations/local_fixtures.json new file mode 100644 index 000000000..0aaf9a808 --- /dev/null +++ b/benefits/core/migrations/local_fixtures.json @@ -0,0 +1,453 @@ +[ + { + "model": "core.pemdata", + "pk": 1, + "fields": { + "label": "(MST) eligibility server public key", + "text_secret_name": null, + "remote_url": "https://raw.githubusercontent.com/cal-itp/eligibility-server/main/keys/server.pub" + } + }, + { + "model": "core.pemdata", + "pk": 2, + "fields": { + "label": "(SBMTD) eligibility server public key", + "text_secret_name": null, + "remote_url": "https://raw.githubusercontent.com/cal-itp/eligibility-server/main/keys/server.pub" + } + }, + { + "model": "core.pemdata", + "pk": 3, + "fields": { + "label": "Benefits client private key", + "text_secret_name": "client-private-key", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 4, + "fields": { + "label": "Benefits client public key", + "text_secret_name": "client-public-key", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 5, + "fields": { + "label": "(MST) payment processor client certificate", + "text_secret_name": "mst-payment-processor-client-cert", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 6, + "fields": { + "label": "(MST) payment processor client certificate private key", + "text_secret_name": "mst-payment-processor-client-cert-private-key", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 7, + "fields": { + "label": "(MST) payment processor client certificate root CA", + "text_secret_name": "mst-payment-processor-client-cert-root-ca", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 8, + "fields": { + "label": "(SacRT) payment processor client certificate", + "text_secret_name": "sacrt-payment-processor-client-cert", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 9, + "fields": { + "label": "(SacRT) payment processor client certificate private key", + "text_secret_name": "sacrt-payment-processor-client-cert-private-key", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 10, + "fields": { + "label": "(SacRT) payment processor client certificate root CA", + "text_secret_name": "sacrt-payment-processor-client-cert-root-ca", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 11, + "fields": { + "label": "(SBMTD) payment processor client certificate", + "text_secret_name": "sbmtd-payment-processor-client-cert", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 12, + "fields": { + "label": "(SBMTD) payment processor client certificate private key", + "text_secret_name": "sbmtd-payment-processor-client-cert-private-key", + "remote_url": null + } + }, + { + "model": "core.pemdata", + "pk": 13, + "fields": { + "label": "(SBMTD) payment processor client certificate root CA", + "text_secret_name": "sbmtd-payment-processor-client-cert-root-ca", + "remote_url": null + } + }, + { + "model": "core.authprovider", + "pk": 1, + "fields": { + "sign_out_button_template": "core/includes/button--sign-out--login-gov.html", + "sign_out_link_template": "core/includes/link--sign-out--login-gov.html", + "client_name": "senior-benefits-oauth-client-name", + "client_id_secret_name": "auth-provider-client-id", + "authority": "https://example.com", + "scope": "verify:senior", + "claim": "senior", + "scheme": "dev-cal-itp_benefits" + } + }, + { + "model": "core.authprovider", + "pk": 2, + "fields": { + "sign_out_button_template": "core/includes/button--sign-out--login-gov.html", + "sign_out_link_template": "core/includes/link--sign-out--login-gov.html", + "client_name": "veteran-benefits-oauth-client-name", + "client_id_secret_name": "auth-provider-client-id", + "authority": "https://example.com", + "scope": "verify:veteran", + "claim": "veteran", + "scheme": "vagov" + } + }, + { + "model": "core.eligibilitytype", + "pk": 1, + "fields": { + "name": "senior", + "label": "(MST) Senior Discount", + "group_id": "group123" + } + }, + { + "model": "core.eligibilitytype", + "pk": 2, + "fields": { + "name": "veteran", + "label": "(MST) Veteran Discount", + "group_id": "group123" + } + }, + { + "model": "core.eligibilitytype", + "pk": 3, + "fields": { + "name": "courtesy_card", + "label": "(MST) Courtesy Card Discount", + "group_id": "group123" + } + }, + { + "model": "core.eligibilitytype", + "pk": 4, + "fields": { + "name": "senior", + "label": "(SacRT) Senior Discount", + "group_id": "group123" + } + }, + { + "model": "core.eligibilitytype", + "pk": 5, + "fields": { + "name": "senior", + "label": "(SBMTD) Senior Discount", + "group_id": "group123" + } + }, + { + "model": "core.eligibilitytype", + "pk": 6, + "fields": { + "name": "mobility_pass", + "label": "(SBMTD) Mobility Pass Discount", + "group_id": "group123" + } + }, + { + "model": "core.eligibilityverifier", + "pk": 1, + "fields": { + "name": "(MST) oauth claims via Login.gov", + "active": true, + "api_url": null, + "api_auth_header": null, + "api_auth_key_secret_name": null, + "eligibility_type": 1, + "public_key": null, + "jwe_cek_enc": null, + "jwe_encryption_alg": null, + "jws_signing_alg": null, + "auth_provider": 1, + "selection_label_template": "eligibility/includes/selection-label--senior.html", + "start_template": "eligibility/start--senior.html", + "form_class": null + } + }, + { + "model": "core.eligibilityverifier", + "pk": 2, + "fields": { + "name": "(MST) VA.gov - veteran", + "active": true, + "api_url": null, + "api_auth_header": null, + "api_auth_key_secret_name": null, + "eligibility_type": 2, + "public_key": null, + "jwe_cek_enc": null, + "jwe_encryption_alg": null, + "jws_signing_alg": null, + "auth_provider": 2, + "selection_label_template": "eligibility/includes/selection-label--veteran.html", + "start_template": "eligibility/start--veteran.html", + "form_class": null + } + }, + { + "model": "core.eligibilityverifier", + "pk": 3, + "fields": { + "name": "(MST) eligibility server verifier", + "active": true, + "api_url": "http://server:8000/verify", + "api_auth_header": "X-Server-API-Key", + "api_auth_key_secret_name": "courtesy-card-verifier-api-auth-key", + "eligibility_type": 3, + "public_key": 1, + "jwe_cek_enc": "A256CBC-HS512", + "jwe_encryption_alg": "RSA-OAEP", + "jws_signing_alg": "RS256", + "auth_provider": null, + "selection_label_template": "eligibility/includes/selection-label--mst-courtesy-card.html", + "start_template": "eligibility/start--mst-courtesy-card.html", + "form_class": "benefits.eligibility.forms.MSTCourtesyCard" + } + }, + { + "model": "core.eligibilityverifier", + "pk": 4, + "fields": { + "name": "(SacRT) oauth claims via Login.gov", + "active": false, + "api_url": null, + "api_auth_header": null, + "api_auth_key_secret_name": null, + "eligibility_type": 4, + "public_key": null, + "jwe_cek_enc": null, + "jwe_encryption_alg": null, + "jws_signing_alg": null, + "auth_provider": 1, + "selection_label_template": "eligibility/includes/selection-label--senior.html", + "start_template": "eligibility/start--senior.html", + "form_class": null + } + }, + { + "model": "core.eligibilityverifier", + "pk": 5, + "fields": { + "name": "(SBMTD) oauth claims via Login.gov", + "active": false, + "api_url": null, + "api_auth_header": null, + "api_auth_key_secret_name": null, + "eligibility_type": 5, + "public_key": null, + "jwe_cek_enc": null, + "jwe_encryption_alg": null, + "jws_signing_alg": null, + "auth_provider": 1, + "selection_label_template": "eligibility/includes/selection-label--senior.html", + "start_template": "eligibility/start--senior.html", + "form_class": null + } + }, + { + "model": "core.eligibilityverifier", + "pk": 6, + "fields": { + "name": "(SBMTD) eligibility server verifier", + "active": true, + "api_url": "http://server:8000/verify", + "api_auth_header": "X-Server-API-Key", + "api_auth_key_secret_name": "mobility-pass-verifier-api-auth-key", + "eligibility_type": 6, + "public_key": 2, + "jwe_cek_enc": "A256CBC-HS512", + "jwe_encryption_alg": "RSA-OAEP", + "jws_signing_alg": "RS256", + "auth_provider": null, + "selection_label_template": "eligibility/includes/selection-label--sbmtd-mobility-pass.html", + "start_template": "eligibility/start--sbmtd-mobility-pass.html", + "form_class": "benefits.eligibility.forms.SBMTDMobilityPass" + } + }, + { + "model": "core.paymentprocessor", + "pk": 1, + "fields": { + "name": "(MST) test payment processor", + "api_base_url": "http://server:8000", + "api_access_token_endpoint": "access-token", + "api_access_token_request_key": "request_access", + "api_access_token_request_val": "REQUEST_ACCESS", + "card_tokenize_url": "http://server:8000/static/tokenize.js", + "card_tokenize_func": "tokenize", + "card_tokenize_env": "test", + "client_cert": 5, + "client_cert_private_key": 6, + "client_cert_root_ca": 7, + "customer_endpoint": "customer", + "customers_endpoint": "customers", + "group_endpoint": "group" + } + }, + { + "model": "core.paymentprocessor", + "pk": 2, + "fields": { + "name": "(SacRT) test payment processor", + "api_base_url": "http://server:8000", + "api_access_token_endpoint": "access-token", + "api_access_token_request_key": "request_access", + "api_access_token_request_val": "REQUEST_ACCESS", + "card_tokenize_url": "http://server:8000/static/tokenize.js", + "card_tokenize_func": "tokenize", + "card_tokenize_env": "test", + "client_cert": 8, + "client_cert_private_key": 9, + "client_cert_root_ca": 10, + "customer_endpoint": "customer", + "customers_endpoint": "customers", + "group_endpoint": "group" + } + }, + { + "model": "core.paymentprocessor", + "pk": 3, + "fields": { + "name": "(SBMTD) test payment processor", + "api_base_url": "http://server:8000", + "api_access_token_endpoint": "access-token", + "api_access_token_request_key": "request_access", + "api_access_token_request_val": "REQUEST_ACCESS", + "card_tokenize_url": "http://server:8000/static/tokenize.js", + "card_tokenize_func": "tokenize", + "card_tokenize_env": "test", + "client_cert": 11, + "client_cert_private_key": 12, + "client_cert_root_ca": 13, + "customer_endpoint": "customer", + "customers_endpoint": "customers", + "group_endpoint": "group" + } + }, + { + "model": "core.transitagency", + "pk": 1, + "fields": { + "slug": "mst", + "short_name": "MST (local)", + "long_name": "Monterey-Salinas Transit (local)", + "agency_id": "mst", + "merchant_id": "mst", + "info_url": "https://mst.org/benefits", + "phone": "888-678-2871", + "active": true, + "payment_processor": 1, + "private_key": 3, + "public_key": 4, + "jws_signing_alg": "RS256", + "index_template": "core/index--mst.html", + "eligibility_index_template": "eligibility/index--mst.html", + "enrollment_success_template": "enrollment/success--mst.html", + "help_template": "core/includes/help--mst.html", + "eligibility_types": [1, 2, 3], + "eligibility_verifiers": [1, 2, 3] + } + }, + { + "model": "core.transitagency", + "pk": 2, + "fields": { + "slug": "sacrt", + "short_name": "SacRT (local)", + "long_name": "Sacramento Regional Transit (local)", + "agency_id": "sacrt", + "merchant_id": "sacrt", + "info_url": "https://sacrt.com/", + "phone": "916-321-2877", + "active": true, + "payment_processor": 2, + "private_key": 3, + "public_key": 4, + "jws_signing_alg": "RS256", + "index_template": "core/index--sacrt.html", + "eligibility_index_template": "eligibility/index--sacrt.html", + "enrollment_success_template": "enrollment/success--sacrt.html", + "help_template": null, + "eligibility_types": [4], + "eligibility_verifiers": [4] + } + }, + { + "model": "core.transitagency", + "pk": 3, + "fields": { + "slug": "sbmtd", + "short_name": "SBMTD (local)", + "long_name": "Santa Barbara MTD (local)", + "agency_id": "sbmtd", + "merchant_id": "sbmtd", + "info_url": "https://sbmtd.gov/taptoride/", + "phone": "805-963-3366", + "active": true, + "payment_processor": 3, + "private_key": 3, + "public_key": 4, + "jws_signing_alg": "RS256", + "index_template": "core/index--sbmtd.html", + "eligibility_index_template": "eligibility/index--sbmtd.html", + "enrollment_success_template": "enrollment/success--sbmtd.html", + "help_template": "core/includes/help--sbmtd.html", + "eligibility_types": [5, 6], + "eligibility_verifiers": [5, 6] + } + } +] diff --git a/benefits/core/migrations/sample_agency.json b/benefits/core/migrations/sample_agency.json deleted file mode 100644 index 6060e4980..000000000 --- a/benefits/core/migrations/sample_agency.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "slug": "mst", - "short_name": "MST (sample)", - "long_name": "Monterey-Salinas Transit (sample)", - "agency_id": "mst", - "merchant_id": "mst", - "info_url": "https://mst.org/benefits", - "phone": "888-678-2871" -} diff --git a/bin/reset_db.sh b/bin/reset_db.sh old mode 100644 new mode 100755 index 126500c3a..b852e5586 --- a/bin/reset_db.sh +++ b/bin/reset_db.sh @@ -19,3 +19,7 @@ bin/init.sh # DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD python manage.py createsuperuser --no-input + +# load sample data fixtures + +python manage.py loaddata benefits/core/migrations/local_fixtures.json diff --git a/tests/cypress/fixtures/README.md b/tests/cypress/fixtures/README.md index 5fc857fec..023e67a56 100644 --- a/tests/cypress/fixtures/README.md +++ b/tests/cypress/fixtures/README.md @@ -1 +1 @@ -The [user data](users.json) corresponds to [the sample data for the eligibility server](https://github.com/cal-itp/eligibility-server/blob/dev/data/server.json). +The [user data](users.json) corresponds to [the sample data for the eligibility server](https://github.com/cal-itp/eligibility-server/blob/main/data/server.csv). diff --git a/tests/cypress/fixtures/transit-agencies.js b/tests/cypress/fixtures/transit-agencies.js index 6a9843f2e..7f30e7cfc 100644 --- a/tests/cypress/fixtures/transit-agencies.js +++ b/tests/cypress/fixtures/transit-agencies.js @@ -1,4 +1,10 @@ -const agency = require("../../../benefits/core/migrations/sample_agency.json"); -const agencies = [{ fields: agency }]; +// extract the "fields" object from the first TransitAgency model fixture + +const local_fixtures = require("../../../benefits/core/migrations/local_fixtures.json"); +const local_agencies = local_fixtures.filter( + (fixture) => fixture.model == "core.transitagency", +); +const first_agency_model = local_agencies[0]; +const agencies = [{ fields: first_agency_model.fields }]; export default agencies; From 4bac3798e7fd73717f0efb4c51e9c7a5d5b0e581 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 13 Feb 2024 16:14:38 -0800 Subject: [PATCH 12/19] fix(tests): container startup script specifically for Cypress --- .github/workflows/tests-cypress.yml | 3 ++- bin/test_start.sh | 9 +++++++++ docs/tests/automated-tests.md | 16 +++++++++++----- 3 files changed, 22 insertions(+), 6 deletions(-) create mode 100755 bin/test_start.sh diff --git a/.github/workflows/tests-cypress.yml b/.github/workflows/tests-cypress.yml index eaaa1f222..ee8bfb7ba 100644 --- a/.github/workflows/tests-cypress.yml +++ b/.github/workflows/tests-cypress.yml @@ -16,7 +16,8 @@ jobs: - name: Start app run: | cp .env.sample .env - docker compose up --detach client server + docker compose up --detach server + docker compose run --detach --service-ports client bin/test_start.sh - name: Run Cypress tests uses: cypress-io/github-action@v6 diff --git a/bin/test_start.sh b/bin/test_start.sh new file mode 100755 index 000000000..12c0ebeef --- /dev/null +++ b/bin/test_start.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eux + +# container startup script specifically for running Cypress tests +# needs to reset the DB with sample data and then start the app normally + +bin/reset_db.sh + +bin/start.sh diff --git a/docs/tests/automated-tests.md b/docs/tests/automated-tests.md index f8062b93d..1b4e6e894 100644 --- a/docs/tests/automated-tests.md +++ b/docs/tests/automated-tests.md @@ -21,25 +21,31 @@ will install `cypress` and its dependencies on your machine. Make sure to run th If not, [install Node.js](https://nodejs.org/en/download/) locally. -2. Start the the application container: +1. Start the local eligibility verification server: ```bash - docker compose up -d client + docker compose up --detach server ``` -3. Change into the `cypress` directory: +1. Start the the application: + + ```bash + docker compose run --detach --service-ports client bin/test_start.sh + ``` + +1. Change into the `cypress` directory: ```bash cd tests/cypress ``` -4. Install all packages and `cypress`. Verify `cypress` installation succeeds: +1. Install all packages and `cypress`. Verify `cypress` installation succeeds: ```bash npm install ``` -5. Run `cypress` with test environment variables and configuration variables: +1. Run `cypress` with test environment variables and configuration variables: ```bash CYPRESS_baseUrl=http://localhost:8000 npm run cypress:open From aeaa3aac62bfb6e78b16376cf8f98e477e56f463 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Thu, 15 Feb 2024 09:02:18 -0800 Subject: [PATCH 13/19] chore(terraform): remove env vars used in data migration these are no longer relevant as we'll use the admin interface directly to configure --- terraform/app_service.tf | 96 ++++------------------------------------ 1 file changed, 9 insertions(+), 87 deletions(-) diff --git a/terraform/app_service.tf b/terraform/app_service.tf index f50175f95..1d3a39dca 100644 --- a/terraform/app_service.tf +++ b/terraform/app_service.tf @@ -66,10 +66,10 @@ resource "azurerm_linux_web_app" "main" { "REQUESTS_READ_TIMEOUT" = "${local.secret_prefix}requests-read-timeout)", # Django settings - "DJANGO_ALLOWED_HOSTS" = "${local.secret_prefix}django-allowed-hosts)", - "DJANGO_DB_DIR" = "${local.secret_prefix}django-db-dir)", - "DJANGO_DEBUG" = local.is_prod ? null : "${local.secret_prefix}django-debug)", - "DJANGO_LOG_LEVEL" = "${local.secret_prefix}django-log-level)", + "DJANGO_ALLOWED_HOSTS" = "${local.secret_prefix}django-allowed-hosts)", + "DJANGO_DB_DIR" = "${local.secret_prefix}django-db-dir)", + "DJANGO_DEBUG" = local.is_prod ? null : "${local.secret_prefix}django-debug)", + "DJANGO_LOG_LEVEL" = "${local.secret_prefix}django-log-level)", "DJANGO_RECAPTCHA_SECRET_KEY" = local.is_dev ? null : "${local.secret_prefix}django-recaptcha-secret-key)", "DJANGO_RECAPTCHA_SITE_KEY" = local.is_dev ? null : "${local.secret_prefix}django-recaptcha-site-key)", @@ -81,96 +81,18 @@ resource "azurerm_linux_web_app" "main" { # Google SSO for Admin - "GOOGLE_SSO_CLIENT_ID" = "${local.secret_prefix}google-sso-client-id", - "GOOGLE_SSO_PROJECT_ID" = "${local.secret_prefix}google-sso-project-id", - "GOOGLE_SSO_CLIENT_SECRET" = "${local.secret_prefix}google-sso-client-secret", + "GOOGLE_SSO_CLIENT_ID" = "${local.secret_prefix}google-sso-client-id", + "GOOGLE_SSO_PROJECT_ID" = "${local.secret_prefix}google-sso-project-id", + "GOOGLE_SSO_CLIENT_SECRET" = "${local.secret_prefix}google-sso-client-secret", "GOOGLE_SSO_ALLOWABLE_DOMAINS" = "${local.secret_prefix}google-sso-allowable-domains", - "GOOGLE_SSO_STAFF_LIST" = "${local.secret_prefix}google-sso-staff-list", - "GOOGLE_SSO_SUPERUSER_LIST" = "${local.secret_prefix}google-sso-superuser-list" + "GOOGLE_SSO_STAFF_LIST" = "${local.secret_prefix}google-sso-staff-list", + "GOOGLE_SSO_SUPERUSER_LIST" = "${local.secret_prefix}google-sso-superuser-list" # Sentry "SENTRY_DSN" = "${local.secret_prefix}sentry-dsn)", "SENTRY_ENVIRONMENT" = local.env_name, "SENTRY_REPORT_URI" = "${local.secret_prefix}sentry-report-uri)", "SENTRY_TRACES_SAMPLE_RATE" = "${local.secret_prefix}sentry-traces-sample-rate)", - - # Environment variables for data migration - "MST_SENIOR_GROUP_ID" = "${local.secret_prefix}mst-senior-group-id)", - "MST_VETERAN_GROUP_ID" = "${local.secret_prefix}mst-veteran-group-id)", - "MST_COURTESY_CARD_GROUP_ID" = "${local.secret_prefix}mst-courtesy-card-group-id)" - "SACRT_SENIOR_GROUP_ID" = "${local.secret_prefix}sacrt-senior-group-id)" - "SBMTD_SENIOR_GROUP_ID" = "${local.secret_prefix}sbmtd-senior-group-id)", - "SBMTD_MOBILITY_PASS_GROUP_ID" = "${local.secret_prefix}sbmtd-mobility-pass-group-id)" - "MST_SERVER_PUBLIC_KEY_URL" = "${local.secret_prefix}mst-server-public-key-url)" - "SBMTD_SERVER_PUBLIC_KEY_URL" = "${local.secret_prefix}sbmtd-server-public-key-url)" - "AUTH_PROVIDER_AUTHORITY" = "${local.secret_prefix}auth-provider-authority)" - "SENIOR_AUTH_PROVIDER_CLIENT_NAME" = "${local.secret_prefix}senior-auth-provider-client-name)" - "SENIOR_AUTH_PROVIDER_SCOPE" = "${local.secret_prefix}senior-auth-provider-scope)" - "SENIOR_AUTH_PROVIDER_CLAIM" = "${local.secret_prefix}senior-auth-provider-claim)" - "SENIOR_AUTH_PROVIDER_SCHEME" = "${local.secret_prefix}senior-auth-provider-scheme)" - "VETERAN_AUTH_PROVIDER_CLIENT_NAME" = "${local.secret_prefix}veteran-auth-provider-client-name)" - "VETERAN_AUTH_PROVIDER_SCOPE" = "${local.secret_prefix}veteran-auth-provider-scope)" - "VETERAN_AUTH_PROVIDER_CLAIM" = "${local.secret_prefix}veteran-auth-provider-claim)" - "VETERAN_AUTH_PROVIDER_SCHEME" = "${local.secret_prefix}veteran-auth-provider-scheme)" - "MST_SENIOR_VERIFIER_NAME" = "${local.secret_prefix}mst-senior-verifier-name)" - "MST_SENIOR_VERIFIER_ACTIVE" = "${local.secret_prefix}mst-senior-verifier-active)" - "MST_VETERAN_VERIFIER_NAME" = "${local.secret_prefix}mst-veteran-verifier-name)" - "MST_VETERAN_VERIFIER_ACTIVE" = "${local.secret_prefix}mst-veteran-verifier-active)" - "COURTESY_CARD_VERIFIER_NAME" = "${local.secret_prefix}courtesy-card-verifier-name)" - "COURTESY_CARD_VERIFIER_ACTIVE" = "${local.secret_prefix}courtesy-card-verifier-active)" - "COURTESY_CARD_VERIFIER_API_URL" = "${local.secret_prefix}courtesy-card-verifier-api-url)" - "COURTESY_CARD_VERIFIER_API_AUTH_HEADER" = "${local.secret_prefix}courtesy-card-verifier-api-auth-header)" - "COURTESY_CARD_VERIFIER_JWE_CEK_ENC" = "${local.secret_prefix}courtesy-card-verifier-jwe-cek-enc)" - "COURTESY_CARD_VERIFIER_JWE_ENCRYPTION_ALG" = "${local.secret_prefix}courtesy-card-verifier-jwe-encryption-alg)" - "COURTESY_CARD_VERIFIER_JWS_SIGNING_ALG" = "${local.secret_prefix}courtesy-card-verifier-jws-signing-alg)" - "SACRT_SENIOR_VERIFIER_NAME" = "${local.secret_prefix}sacrt-senior-verifier-name)" - "SACRT_SENIOR_VERIFIER_ACTIVE" = "${local.secret_prefix}sacrt-senior-verifier-active)" - "SBMTD_SENIOR_VERIFIER_NAME" = "${local.secret_prefix}sbmtd-senior-verifier-name)" - "SBMTD_SENIOR_VERIFIER_ACTIVE" = "${local.secret_prefix}sbmtd-senior-verifier-active)" - "MST_PAYMENT_PROCESSOR_NAME" = "${local.secret_prefix}mst-payment-processor-name)" - "MST_PAYMENT_PROCESSOR_API_BASE_URL" = "${local.secret_prefix}mst-payment-processor-api-base-url)" - "MST_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_ENDPOINT" = "${local.secret_prefix}mst-payment-processor-api-access-token-endpoint)" - "MST_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_KEY" = "${local.secret_prefix}mst-payment-processor-api-access-token-request-key)" - "MST_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_VAL" = "${local.secret_prefix}mst-payment-processor-api-access-token-request-val)" - "MST_PAYMENT_PROCESSOR_CARD_TOKENIZE_URL" = "${local.secret_prefix}mst-payment-processor-card-tokenize-url)" - "MST_PAYMENT_PROCESSOR_CARD_TOKENIZE_FUNC" = "${local.secret_prefix}mst-payment-processor-card-tokenize-func)" - "MST_PAYMENT_PROCESSOR_CARD_TOKENIZE_ENV" = "${local.secret_prefix}mst-payment-processor-card-tokenize-env)" - "SACRT_PAYMENT_PROCESSOR_NAME" = "${local.secret_prefix}sacrt-payment-processor-name)" - "SACRT_PAYMENT_PROCESSOR_API_BASE_URL" = "${local.secret_prefix}sacrt-payment-processor-api-base-url)" - "SACRT_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_ENDPOINT" = "${local.secret_prefix}sacrt-payment-processor-api-access-token-endpoint)" - "SACRT_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_KEY" = "${local.secret_prefix}sacrt-payment-processor-api-access-token-request-key)" - "SACRT_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_VAL" = "${local.secret_prefix}sacrt-payment-processor-api-access-token-request-val)" - "SACRT_PAYMENT_PROCESSOR_CARD_TOKENIZE_URL" = "${local.secret_prefix}sacrt-payment-processor-card-tokenize-url)" - "SACRT_PAYMENT_PROCESSOR_CARD_TOKENIZE_FUNC" = "${local.secret_prefix}sacrt-payment-processor-card-tokenize-func)" - "SACRT_PAYMENT_PROCESSOR_CARD_TOKENIZE_ENV" = "${local.secret_prefix}sacrt-payment-processor-card-tokenize-env)" - "SBMTD_PAYMENT_PROCESSOR_NAME" = "${local.secret_prefix}sbmtd-payment-processor-name)" - "SBMTD_PAYMENT_PROCESSOR_API_BASE_URL" = "${local.secret_prefix}sbmtd-payment-processor-api-base-url)" - "SBMTD_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_ENDPOINT" = "${local.secret_prefix}sbmtd-payment-processor-api-access-token-endpoint)" - "SBMTD_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_KEY" = "${local.secret_prefix}sbmtd-payment-processor-api-access-token-request-key)" - "SBMTD_PAYMENT_PROCESSOR_API_ACCESS_TOKEN_REQUEST_VAL" = "${local.secret_prefix}sbmtd-payment-processor-api-access-token-request-val)" - "SBMTD_PAYMENT_PROCESSOR_CARD_TOKENIZE_URL" = "${local.secret_prefix}sbmtd-payment-processor-card-tokenize-url)" - "SBMTD_PAYMENT_PROCESSOR_CARD_TOKENIZE_FUNC" = "${local.secret_prefix}sbmtd-payment-processor-card-tokenize-func)" - "SBMTD_PAYMENT_PROCESSOR_CARD_TOKENIZE_ENV" = "${local.secret_prefix}sbmtd-payment-processor-card-tokenize-env)" - "MOBILITY_PASS_VERIFIER_NAME" = "${local.secret_prefix}mobility-pass-verifier-name)" - "MOBILITY_PASS_VERIFIER_ACTIVE" = "${local.secret_prefix}mobility-pass-verifier-active)" - "MOBILITY_PASS_VERIFIER_API_URL" = "${local.secret_prefix}mobility-pass-verifier-api-url)" - "MOBILITY_PASS_VERIFIER_API_AUTH_HEADER" = "${local.secret_prefix}mobility-pass-verifier-api-auth-header)" - "MOBILITY_PASS_VERIFIER_JWE_CEK_ENC" = "${local.secret_prefix}mobility-pass-verifier-jwe-cek-enc)" - "MOBILITY_PASS_VERIFIER_JWE_ENCRYPTION_ALG" = "${local.secret_prefix}mobility-pass-verifier-jwe-encryption-alg)" - "MOBILITY_PASS_VERIFIER_JWS_SIGNING_ALG" = "${local.secret_prefix}mobility-pass-verifier-jws-signing-alg)" - "MST_AGENCY_SHORT_NAME" = "${local.secret_prefix}mst-agency-short-name)" - "MST_AGENCY_LONG_NAME" = "${local.secret_prefix}mst-agency-long-name)" - "MST_AGENCY_JWS_SIGNING_ALG" = "${local.secret_prefix}mst-agency-jws-signing-alg)" - "SACRT_AGENCY_SHORT_NAME" = "${local.secret_prefix}sacrt-agency-short-name)" - "SACRT_AGENCY_LONG_NAME" = "${local.secret_prefix}sacrt-agency-long-name)" - "SACRT_AGENCY_MERCHANT_ID" = "${local.secret_prefix}sacrt-agency-merchant-id)" - "SACRT_AGENCY_ACTIVE" = "${local.secret_prefix}sacrt-agency-active)" - "SACRT_AGENCY_JWS_SIGNING_ALG" = "${local.secret_prefix}sacrt-agency-jws-signing-alg)" - "SBMTD_AGENCY_SHORT_NAME" = "${local.secret_prefix}sbmtd-agency-short-name)" - "SBMTD_AGENCY_LONG_NAME" = "${local.secret_prefix}sbmtd-agency-long-name)" - "SBMTD_AGENCY_MERCHANT_ID" = "${local.secret_prefix}sbmtd-agency-merchant-id)" - "SBMTD_AGENCY_ACTIVE" = "${local.secret_prefix}sbmtd-agency-active)" - "SBMTD_AGENCY_JWS_SIGNING_ALG" = "${local.secret_prefix}sbmtd-agency-jws-signing-alg)" } storage_account { From b71b52581c663f421eb195a52fe3d354eee21797 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Thu, 15 Feb 2024 21:43:55 +0000 Subject: [PATCH 14/19] refactor(rest_db): allow more local customization * devs may or may not want to reset their local DB * devs may want to change which DB file is targeted * devs may want to change which fixture file is loaded update docs to reflect these changes --- .env.sample | 6 +++ benefits/settings.py | 2 +- bin/reset_db.sh | 58 ++++++++++++--------- docs/configuration/data.md | 43 +++++++-------- docs/configuration/environment-variables.md | 50 +++++++++++++++++- 5 files changed, 107 insertions(+), 52 deletions(-) diff --git a/.env.sample b/.env.sample index c79424341..fb14ecd8e 100644 --- a/.env.sample +++ b/.env.sample @@ -1,6 +1,12 @@ DJANGO_SUPERUSER_USERNAME=benefits-admin DJANGO_SUPERUSER_EMAIL=benefits-admin@calitp.org DJANGO_SUPERUSER_PASSWORD=superuser12345! + +DJANGO_DB_RESET=true +DJANGO_DB_DIR=. +DJANGO_DB_FILE=django.db +DJANGO_DB_FIXTURES="benefits/core/migrations/local_fixtures.json" + testsecret=Hello from the local environment! auth_provider_client_id=benefits-oauth-client-id courtesy_card_verifier_api_auth_key=server-auth-token diff --git a/benefits/settings.py b/benefits/settings.py index 0ce3ffdde..ce3f7eeb5 100644 --- a/benefits/settings.py +++ b/benefits/settings.py @@ -166,7 +166,7 @@ def RUNTIME_ENVIRONMENT(): DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(DATABASE_DIR, "django.db"), + "NAME": os.path.join(DATABASE_DIR, os.environ.get("DJANGO_DB_FILE", "django.db")), } } diff --git a/bin/reset_db.sh b/bin/reset_db.sh index b852e5586..26529338a 100755 --- a/bin/reset_db.sh +++ b/bin/reset_db.sh @@ -1,25 +1,35 @@ #!/usr/bin/env bash -set -eux - -# remove database file - -# construct the path to the database file from environment or default -DB_DIR="${DJANGO_DB_DIR:-.}" -DB_FILE="${DB_DIR}/django.db" - -# -f forces the delete (and avoids an error when the file doesn't exist) -rm -f "${DB_FILE}" - -# run database migrations and other initialization - -bin/init.sh - -# create a superuser account for backend admin access -# set username, email, and password using environment variables -# DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD - -python manage.py createsuperuser --no-input - -# load sample data fixtures - -python manage.py loaddata benefits/core/migrations/local_fixtures.json +set -ex + +# whether to reset database file, defaults to true +DB_RESET="${DJANGO_DB_RESET:-true}" +# optional fixtures to import +FIXTURES="${DJANGO_DB_FIXTURES}" + +if [[ $DB_RESET = true ]]; then + # construct the path to the database file from environment or default + DB_DIR="${DJANGO_DB_DIR:-.}" + DB_FILE="${DJANGO_DB_FILE:-django.db}" + DB_PATH="${DB_DIR}/${DB_FILE}" + + rm -f "${DB_PATH}" + + # run database migrations and other initialization + bin/init.sh + + # create a superuser account for backend admin access + # set username, email, and password using environment variables + # DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD + python manage.py createsuperuser --no-input +else + echo "DB_RESET is false, skipping" +fi + +valid_fixtures=$( echo $FIXTURES | grep -e fixtures\.json$ ) + +if [[ -n "$valid_fixtures" ]]; then + # load data fixtures + python manage.py loaddata "$FIXTURES" +else + echo "No JSON fixtures to load" +fi diff --git a/docs/configuration/data.md b/docs/configuration/data.md index 6e52de0b8..b87c79dea 100644 --- a/docs/configuration/data.md +++ b/docs/configuration/data.md @@ -1,8 +1,8 @@ # Configuration data -!!! example "Data migration file" +!!! example "Sample data fixtures" - [`benefits/core/migrations/0002_data.py`][data-migration] + [`benefits/core/migrations/local_fixtures.json`][sample-fixtures] !!! tldr "Django docs" @@ -10,14 +10,15 @@ ## Introduction -Django [data migrations](https://docs.djangoproject.com/en/4.0/topics/migrations/#data-migrations) are used to load the database with instances of the app's model classes, defined in [`benefits/core/models.py`][core-models]. +The app's model classes are defined in [`benefits/core/models.py`][core-models]. Migrations are run as the application starts up. See the [`bin/init.sh`][init] script. The sample values provided in the repository are sufficient to run the app locally and interact with e.g. the sample Transit -Agencies. +Agencies. [Django fixtures][django-fixtures] are used to load the database with sample data when running locally. -During the [deployment](../deployment/README.md) process, environment-specific values are set in environment variables and are read by the data migration file to build that environment's configuration database. See the [data migration file][data-migration] for the environment variable names. +During the [deployment](../deployment/README.md) process, some environment-specific values are set in environment variables and +read dynamically at runtime. Most configuration values are managed directly in the Django Admin interface at the `/admin` endpoint. ## Sample data @@ -37,32 +38,24 @@ Some configuration data is not available with the samples in the repository: - Payment processor configuration for the enrollment phase - Amplitude configuration for capturing analytics events -### Sample transit agency: `ABC` +## Rebuilding the configuration database locally -- Presents the user a choice between two different eligibility pathways -- One eligibility verifier requires authentication -- One eligibility verifier does not require authentication +A local Django database will be initialized upon first startup of the devcontainer. -### Sample transit agency: `DefTL` - -- Single eligibility pathway, no choice presented to the user -- Eligibility verifier does not require authentication - -## Building the configuration database - -When the data migration changes, the configuration database needs to be rebuilt. - -The file is called `django.db` and the following commands will rebuild it. - -Run these commands from within the repository root, inside the devcontainer: +To rebuild the local Django database, run the [`bin/reset_db.sh`][reset-db] script from within the repository root, +inside the devcontainer: ```bash -bin/init.sh +bin/reset_db.sh ``` +See the [Django Environment Variables](environment-variables.md#django) section for details about how to configure the local +database rebuild. + [core-models]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/models.py -[django-load-initial-data]: https://docs.djangoproject.com/en/4.0/howto/initial-data/ +[django-fixtures]: https://docs.djangoproject.com/en/5.0/topics/db/fixtures/ +[django-load-initial-data]: https://docs.djangoproject.com/en/5.0/howto/initial-data/ [eligibility-server]: https://docs.calitp.org/eligibility-server -[data-migration]: https://github.com/cal-itp/benefits/tree/dev/benefits/core/migrations/0002_data.py -[helper-migration]: https://github.com/cal-itp/benefits/tree/dev/benefits/core/migrations/0003_data_migration_order.py [init]: https://github.com/cal-itp/benefits/blob/dev/bin/init.sh +[reset-db]: https://github.com/cal-itp/benefits/blob/dev/bin/reset_db.sh +[sample-fixtures]: https://github.com/cal-itp/benefits/tree/dev/benefits/core/migrations/local_fixtures.json diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 305131cf3..455a637b1 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -2,7 +2,8 @@ The first steps of the Getting Started guide mention [creating an `.env` file][getting-started_create-env]. -The sections below outline in more detail the application environment variables that you may want to override, and their purpose. In App Service, this is more generally called the ["configuration"][app-service-config]. +The sections below outline in more detail the application environment variables that you may want to override, and their purpose. +In Azure App Services, this is more generally called the ["configuration"][app-service-config]. See other topic pages in this section for more specific environment variable configurations. @@ -71,6 +72,39 @@ writable by the Django process._ By default, the base project directory (i.e. the root of the repository). +### `DJANGO_DB_FILE` + +!!! info "Local configuration" + + This setting only affects the app running on localhost + +The name of the Django database file to use locally (during both normal app startup and for resetting the database). + +By default, `django.db`. + +### `DJANGO_DB_FIXTURES` + +!!! info "Local configuration" + + This setting only affects the app running on localhost + +A path, relative to the repository root, of Django data fixtures to load when resetting the database. + +The file must end in `fixtures.json` for the script to process it correctly. + +By default, `benefits/core/migrations/local_fixtures.json`. + +### `DJANGO_DB_RESET` + +!!! info "Local configuration" + + This setting only affects the app running on localhost + +Boolean: + +- `True` (default): deletes the existing database file and runs fresh Django migrations. +- `False`: Django uses the existing database file. + ### `DJANGO_DEBUG` !!! warning "Deployment configuration" @@ -79,7 +113,7 @@ By default, the base project directory (i.e. the root of the repository). !!! tldr "Django docs" - [Settings: `DEBUG`](https://docs.djangoproject.com/en/4.0/ref/settings/#debug) + [Settings: `DEBUG`](https://docs.djangoproject.com/en/5.0/ref/settings/#debug) Boolean: @@ -128,14 +162,26 @@ Django's primary secret, keep this safe! ### `DJANGO_SUPERUSER_EMAIL` +!!! info "Local configuration" + + This setting only affects the app running on localhost + The email address of the Django Admin superuser created when resetting the database. ### `DJANGO_SUPERUSER_PASSWORD` +!!! info "Local configuration" + + This setting only affects the app running on localhost + The password of the Django Admin superuser created when resetting the database. ### `DJANGO_SUPERUSER_USERNAME` +!!! info "Local configuration" + + This setting only affects the app running on localhost + The username of the Django Admin superuser created when resetting the database. ### `DJANGO_TRUSTED_ORIGINS` From 0a562fb8e181b6afb8635bd68d041cde55312ac9 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Thu, 15 Feb 2024 22:28:45 +0000 Subject: [PATCH 15/19] feat(git): ignore fixtures except the included sample --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 46b0f2ec3..228504a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ *.db *.env +*fixtures.json +!benefits/core/migrations/local_fixtures.json *.mo *.tfbackend *.tmp From 94312c019717065c78930a6b675279806c3d8bd4 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 20 Feb 2024 12:54:20 -0800 Subject: [PATCH 16/19] docs(deployment): update language around config database note the use of Key Vault for secrets, Django admin for non-secrets --- docs/deployment/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/deployment/README.md b/docs/deployment/README.md index 697325fae..c3e7046cc 100644 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -1,6 +1,6 @@ # Overview -[dev-benefits.calitp.org][dev-benefits] is currently deployed into a Microsoft Azure account provided by [California Department of Technology (CDT)'s Office of Enterprise Technology (OET)][oet], a.k.a. the "DevSecOps" team. More specifically, it uses [custom containers][app-service-containers] on [Azure App Service][app-service]. [More about the infrastructure.](infrastructure.md) +The Benefits app is currently deployed into a Microsoft Azure account provided by [California Department of Technology (CDT)'s Office of Enterprise Technology (OET)][oet], a.k.a. the "DevSecOps" team. More specifically, it uses [custom containers][app-service-containers] on [Azure App Service][app-service]. [More about the infrastructure.](infrastructure.md) ## Deployment process @@ -20,18 +20,18 @@ You can view what Git commit is deployed for a given environment by visitng the ## Configuration -[Configuration settings](../configuration/README.md) are stored as Application Configuration variables in Azure. -[Data](../configuration/data.md) is loaded via Django data migrations. +Sensitive [configuration settings](../configuration/README.md) are maintained as Application Configuration variables in Azure, +referencing [Azure Key Vault secrets](https://azure.microsoft.com/en-us/products/key-vault/). Other non-sensitive configuration +is maintained directly in the configuration database via the [Django Admin](https://docs.djangoproject.com/en/5.0/ref/contrib/admin/). ## Docker images Docker images for each of the deploy branches are available from GitHub Container Registry (GHCR): -* [Repository Package page](https://github.com/cal-itp/benefits/pkgs/container/benefits) -* Image path: `ghcr.io/cal-itp/benefits` -* Image tags: `dev`, `test`, `prod` +- [Repository Package page](https://github.com/cal-itp/benefits/pkgs/container/benefits) +- Image path: `ghcr.io/cal-itp/benefits` +- Image tags: `dev`, `test`, `prod` -[dev-benefits]: https://dev-benefits.calitp.org [oet]: https://techblog.cdt.ca.gov/2020/06/cdt-taking-the-lead-in-digital-transformation/ [app-service-containers]: https://docs.microsoft.com/en-us/azure/app-service/configure-custom-container [app-service]: https://docs.microsoft.com/en-us/azure/app-service/overview From e37ed91b39a2002fbc7c3508e5183b9dc4839bf8 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 20 Feb 2024 12:55:28 -0800 Subject: [PATCH 17/19] refactor(migrations): update helper script and docs now that we have the Admin interface, we don't want to regrenerate the existing migration rather, we need to generate new migrations each time to reflect model changes into the DB --- bin/makemigrations.sh | 19 +------------------ docs/development/models-migrations.md | 23 ++++++----------------- 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/bin/makemigrations.sh b/bin/makemigrations.sh index 4fc8508f0..89d755f0b 100755 --- a/bin/makemigrations.sh +++ b/bin/makemigrations.sh @@ -1,27 +1,10 @@ #!/usr/bin/env bash set -eux -# create temporary directory (if it doesn't already exist) - -mkdir -p benefits/core/old_migrations - -# move old migrations to temporary directory, but keep init file - -mv benefits/core/migrations/* benefits/core/old_migrations -cp benefits/core/old_migrations/__init__.py benefits/core/migrations - -# regenerate +# generate python manage.py makemigrations -# copy over migrations that don't exist - -cp benefits/core/old_migrations/* benefits/core/migrations --no-clobber --recursive - -# clean up temporary directory - -rm -rf benefits/core/old_migrations - # reformat with black python -m black benefits/core/migrations/*.py diff --git a/docs/development/models-migrations.md b/docs/development/models-migrations.md index 413289b9c..631fcc33d 100644 --- a/docs/development/models-migrations.md +++ b/docs/development/models-migrations.md @@ -6,24 +6,17 @@ [`benefits/core/migrations/0001_initial.py`][core-migrations] - [`benefits/core/migrations/0002_data.py`][data-migrations] - Cal-ITP Benefits defines a number of [models][core-models] in the core application, used throughout the codebase to configure different parts of the UI and logic. -The Cal-ITP Benefits database is a simple read-only Sqlite database, initialized from the [data migration](../configuration/data.md) files. - -## Migrations - -The database is rebuilt from scratch each time the container starts. We maintain a few [migration][migrations] files that set up the schema and load initial data. - -These files always represent the current schema and data for the database and match the current structure of the model classes. +The Cal-ITP Benefits database is a simple Sqlite database that mostly acts as a read-only configuration store. +Runtime configuration changes can be persisted via [Django's Admin interface](https://docs.djangoproject.com/en/5.0/ref/contrib/admin/). ## Updating models -When models are updated, the migration should be updated as well. +When models are updated, new migrations must be generated to reflect those changes into the configuration database. -A simple helper script exists to regenerate the migration file based on the current state of models in the local directory: +A simple helper script exists to generate migrations based on the current state of models in the local directory: [`bin/makemigrations.sh`][makemigrations] @@ -33,15 +26,11 @@ bin/makemigrations.sh This script: -1. Copies the existing migration files to a temporary directory 1. Runs the django `makemigrations` command -1. Copies back any migration files that are missing (data migration file) -1. Formats the newly regenerated schema migration file with `black` +1. Formats the newly regenerated migration file with `black` -This will result in a simple diff of changes on the schema migration file. Commit these changes (including the timestamp!) along with the model changes. +Commit the new migration file along with the model changes. [core-models]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/models.py [core-migrations]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/migrations/0001_initial.py -[data-migrations]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/migrations/0002_data.py [makemigrations]: https://github.com/cal-itp/benefits/blob/dev/bin/makemigrations.sh -[migrations]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/migrations From d6bf990ea2db4b9458371c7aa709e1ec40389c29 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 20 Feb 2024 13:00:21 -0800 Subject: [PATCH 18/19] chore(docs): update references to Django docs we use Django 5.x now --- benefits/settings.py | 2 +- benefits/urls.py | 2 +- docs/README.md | 2 +- docs/configuration/README.md | 8 ++++---- docs/configuration/environment-variables.md | 8 ++++---- docs/development/i18n.md | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/benefits/settings.py b/benefits/settings.py index ce3f7eeb5..35163a339 100644 --- a/benefits/settings.py +++ b/benefits/settings.py @@ -124,7 +124,7 @@ def RUNTIME_ENVIRONMENT(): # SSL terminates before getting to Django, and NGINX adds this header to indicate # if the original request was secure or not # -# See https://docs.djangoproject.com/en/4.0/ref/settings/#secure-proxy-ssl-header +# See https://docs.djangoproject.com/en/5.0/ref/settings/#secure-proxy-ssl-header if not DEBUG: SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") diff --git a/benefits/urls.py b/benefits/urls.py index 0a5d658ec..30d95f018 100644 --- a/benefits/urls.py +++ b/benefits/urls.py @@ -2,7 +2,7 @@ benefits URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/4.0/topics/http/urls/ + https://docs.djangoproject.com/en/5.0/topics/http/urls/ """ import logging diff --git a/docs/README.md b/docs/README.md index 0c7405759..d7a01dbbc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -96,4 +96,4 @@ All code changes are reviewed by at least one other member of the engineering te [interconnections]: deployment/infrastructure/#system-interconnections [hosting]: deployment/ [littlepay]: https://littlepay.com/ -[i18n]: https://docs.djangoproject.com/en/4.0/topics/i18n/ +[i18n]: https://docs.djangoproject.com/en/5.0/topics/i18n/ diff --git a/docs/configuration/README.md b/docs/configuration/README.md index ef12dcede..9ed629e16 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -13,7 +13,7 @@ startup. The model objects defined in the data migration file are also loaded into and seed Django's database at application startup time. - See the [Setting secrets](../deployment/secrets) section for how to set secret values for a deployment. +See the [Setting secrets](../deployment/secrets) section for how to set secret values for a deployment. ## Django settings @@ -77,9 +77,9 @@ else: [benefits-manage]: https://github.com/cal-itp/benefits/blob/dev/manage.py [benefits-settings]: https://github.com/cal-itp/benefits/blob/dev/benefits/settings.py [benefits-wsgi]: https://github.com/cal-itp/benefits/blob/dev/benefits/wsgi.py -[django-model]: https://docs.djangoproject.com/en/4.0/topics/db/models/ -[django-settings]: https://docs.djangoproject.com/en/4.0/topics/settings/ -[django-using-settings]: https://docs.djangoproject.com/en/4.0/topics/settings/#using-settings-in-python-code +[django-model]: https://docs.djangoproject.com/en/5.0/topics/db/models/ +[django-settings]: https://docs.djangoproject.com/en/5.0/topics/settings/ +[django-using-settings]: https://docs.djangoproject.com/en/5.0/topics/settings/#using-settings-in-python-code [env-vars]: environment-variables.md [data]: data.md [getting-started]: ../getting-started/README.md diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 455a637b1..cd150c283 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -57,7 +57,7 @@ If blank or an invalid key, analytics events aren't captured (though may still b !!! tldr "Django docs" - [Settings: `ALLOWS_HOSTS`](https://docs.djangoproject.com/en/4.0/ref/settings/#allowed-hosts) + [Settings: `ALLOWS_HOSTS`](https://docs.djangoproject.com/en/5.0/ref/settings/#allowed-hosts) A list of strings representing the host/domain names that this Django site can serve. @@ -142,7 +142,7 @@ From inside the container, the app is always listening on port `8000`. !!! tldr "Django docs" - [Settings: `LOGGING_CONFIG`](https://docs.djangoproject.com/en/4.0/ref/settings/#logging-config) + [Settings: `LOGGING_CONFIG`](https://docs.djangoproject.com/en/5.0/ref/settings/#logging-config) The log level used in the application's logging configuration. @@ -156,7 +156,7 @@ By default the application sends logs to `stdout`. !!! tldr "Django docs" - [Settings: `SECRET_KEY`](https://docs.djangoproject.com/en/4.0/ref/settings/#secret-key) + [Settings: `SECRET_KEY`](https://docs.djangoproject.com/en/5.0/ref/settings/#secret-key) Django's primary secret, keep this safe! @@ -192,7 +192,7 @@ The username of the Django Admin superuser created when resetting the database. !!! tldr "Django docs" - [Settings: `CSRF_TRUSTED_ORIGINS`](https://docs.djangoproject.com/en/4.0/ref/settings/#csrf-trusted-origins) + [Settings: `CSRF_TRUSTED_ORIGINS`](https://docs.djangoproject.com/en/5.0/ref/settings/#csrf-trusted-origins) Comma-separated list of hosts which are trusted origins for unsafe requests (e.g. POST) diff --git a/docs/development/i18n.md b/docs/development/i18n.md index 093a7e102..4dfcfb24e 100644 --- a/docs/development/i18n.md +++ b/docs/development/i18n.md @@ -2,9 +2,9 @@ !!! tldr "Django docs" - [Internationalization and localization](https://docs.djangoproject.com/en/4.0/topics/i18n/) + [Internationalization and localization](https://docs.djangoproject.com/en/5.0/topics/i18n/) - [Translation](https://docs.djangoproject.com/en/4.0/topics/i18n/translation/) + [Translation](https://docs.djangoproject.com/en/5.0/topics/i18n/translation/) !!! example "Message files" @@ -12,7 +12,7 @@ The Cal-ITP Benefits application is fully internationalized and available in both English and Spanish. -It uses Django's built-in support for translation using [message files](https://docs.djangoproject.com/en/4.0/topics/i18n/#term-message-file), which contain entries of `msgid`/`msgstr` pairs. The `msgid` is referenced in source code so that Django takes care of showing the `msgstr` for the user's language. +It uses Django's built-in support for translation using [message files](https://docs.djangoproject.com/en/5S.0/topics/i18n/#term-message-file), which contain entries of `msgid`/`msgstr` pairs. The `msgid` is referenced in source code so that Django takes care of showing the `msgstr` for the user's language. ## Updating message files @@ -42,7 +42,7 @@ When templates have different copy per agency, create a new template for that ag ### Fuzzy strings -From [Django docs](https://docs.djangoproject.com/en/4.0/topics/i18n/translation/#message-files): +From [Django docs](https://docs.djangoproject.com/en/5.0/topics/i18n/translation/#message-files): > `makemessages` sometimes generates translation entries marked as fuzzy, e.g. when translations are inferred from previously translated strings. From 0f8ab2041e72c6b252daa9f9b3b92124548c32c2 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 23 Feb 2024 09:49:12 -0800 Subject: [PATCH 19/19] docs: fix typo in Django URL Co-authored-by: Angela Tran --- docs/development/i18n.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/i18n.md b/docs/development/i18n.md index 4dfcfb24e..7261011bd 100644 --- a/docs/development/i18n.md +++ b/docs/development/i18n.md @@ -12,7 +12,7 @@ The Cal-ITP Benefits application is fully internationalized and available in both English and Spanish. -It uses Django's built-in support for translation using [message files](https://docs.djangoproject.com/en/5S.0/topics/i18n/#term-message-file), which contain entries of `msgid`/`msgstr` pairs. The `msgid` is referenced in source code so that Django takes care of showing the `msgstr` for the user's language. +It uses Django's built-in support for translation using [message files](https://docs.djangoproject.com/en/5.0/topics/i18n/#term-message-file), which contain entries of `msgid`/`msgstr` pairs. The `msgid` is referenced in source code so that Django takes care of showing the `msgstr` for the user's language. ## Updating message files