diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 9a412adc..3f154ac9 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -13,6 +13,6 @@ jobs: trivy-image-config: "trivy.yaml" juju-channel: 3.1/stable channel: 1.28-strict/stable - modules: '["test_charm", "test_saml", "test_users"]' + modules: '["test_charm", "test_saml", "test_users", "test_db_migration"]' self-hosted-runner: true self-hosted-runner-label: "edge" diff --git a/charmcraft.yaml b/charmcraft.yaml index 5a8c48b2..a41bb190 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -12,5 +12,6 @@ bases: channel: "20.04" parts: charm: + build-packages: [cargo, rustc, pkg-config, libffi-dev, libssl-dev] charm-python-packages: [setuptools, pip] # https://discourse.charmhub.io/t/install-or-update-python-packages-before-packing-a-charm/5158 charm-binary-python-packages: [cosl] # https://github.com/canonical/charmcraft/issues/1269 diff --git a/discourse_rock/patches/db_migrations.patch b/discourse_rock/patches/db_migrations.patch new file mode 100644 index 00000000..57eb6129 --- /dev/null +++ b/discourse_rock/patches/db_migrations.patch @@ -0,0 +1,16 @@ +diff --git a/db/post_migrate/20240212034010_drop_deprecated_columns.rb b/db/post_migrate/20240212034010_drop_deprecated_columns.rb +index 0899da20..015fc6d5 100644 +--- a/db/post_migrate/20240212034010_drop_deprecated_columns.rb ++++ b/db/post_migrate/20240212034010_drop_deprecated_columns.rb +@@ -19,6 +19,11 @@ class DropDeprecatedColumns < ActiveRecord::Migration[7.0] + } + + def up ++ execute <<~SQL ++ DROP TRIGGER IF EXISTS invites_user_id_readonly ON invites; ++ DROP TRIGGER IF EXISTS invites_redeemed_at_readonly ON invites; ++ DROP TRIGGER IF EXISTS user_api_keys_scopes_readonly ON user_api_keys; ++ SQL + DROPPED_COLUMNS.each { |table, columns| Migration::ColumnDropper.execute_drop(table, columns) } + end + diff --git a/discourse_rock/rockcraft.yaml b/discourse_rock/rockcraft.yaml index d6d5916f..27be1bbc 100644 --- a/discourse_rock/rockcraft.yaml +++ b/discourse_rock/rockcraft.yaml @@ -192,6 +192,7 @@ parts: - git after: [discourse, patches] override-stage: | + git -C srv/discourse/app apply patches/db_migrations.patch git -C srv/discourse/app apply patches/lp1903695.patch git -C srv/discourse/app apply patches/discourse-charm.patch git -C srv/discourse/app apply patches/sigterm.patch diff --git a/testing_database/creating-the-testing-database.md b/testing_database/creating-the-testing-database.md new file mode 100644 index 00000000..2d882b37 --- /dev/null +++ b/testing_database/creating-the-testing-database.md @@ -0,0 +1,35 @@ +# Create the testing database +At the writing this document, the testing database is created using Discourse v3.2.0. +To get the same result use Charm revision 162 and resource revision 152. + +## Create the database +First of all we need to deploy the Discourse following the [tutorial](https://github.com/canonical/discourse-k8s-operator/blob/main/docs/tutorial.md). + +Then, we need to create a 2 new users for testing using the actions: + +```juju run discourse-k8s/0 create-user email=email@example.com admin=true``` +```juju run discourse-k8s/0 create-user email=email2@example.com``` + +Please note that the first user is an admin and the second one is not. Also please not the passwords that are generated automatically by the command. + +Now open the Discourse URL in a browser, login with the first user (admin) and create a new topic. Reply to this topic as the admin user again. Then, login with the second user and reply to this topic. Then login with the first user and approve the second users reply. + +## Export the database + +First we need to get the database password: +```juju run postgresql-k8s/0 get-password username=operator``` + +Ssh into the database +```juju ssh --container postgresql postgresql-k8s/0 bash``` + +Create a folder to dump the db +```mkdir -p /srv/dump/``` + +Dump the db. Ip here is the unit ip +```pg_dump -Fc -h 10.1.187.134 -U operator -d discourse > "/srv/dump/testing_database.sql"``` + +Exit the container +```exit``` + +Copy the dump into local file system. +```juju scp --container postgresql postgresql-k8s/0:/srv/dump/testing_database.sql./testing_database.sql``` diff --git a/testing_database/testing_database.sql b/testing_database/testing_database.sql new file mode 100644 index 00000000..ee758b44 Binary files /dev/null and b/testing_database/testing_database.sql differ diff --git a/tests/integration/test_db_migration.py b/tests/integration/test_db_migration.py new file mode 100644 index 00000000..5197dc96 --- /dev/null +++ b/tests/integration/test_db_migration.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +"""Discourse integration tests.""" + +import logging + +import pytest +from botocore.config import Config +from ops.model import WaitingStatus +from pytest_operator.plugin import Model, OpsTest + +logger = logging.getLogger(__name__) + + +@pytest.mark.asyncio +@pytest.mark.abort_on_fail +async def test_db_migration(model: Model, ops_test: OpsTest, pytestconfig: Config, run_action): + """ + arrange: preload postgres with a testing db that is created in Discourse v3.2.0 + act: deploy and integrate with Discourse v3.3.0 (latest) + assert: discourse is active/idle + + Discourse must be active idle, it might create migration errors related to + not being able to delete some columns because of triggers. This is fixed + with a patch but this patch only works for Discourse v3.2.0 and we might + need to create a new patch for the new version of Discourse. + """ + postgres_app = await model.deploy( + "postgresql-k8s", + channel="14/stable", + series="jammy", + revision=300, + trust=True, + config={"profile": "testing"}, + ) + async with ops_test.fast_forward(): + await model.wait_for_idle(apps=[postgres_app.name], status="active") + await postgres_app.set_config( + { + "plugin_hstore_enable": "true", + "plugin_pg_trgm_enable": "true", + } + ) + await model.wait_for_idle(apps=[postgres_app.name], status="active") + db_pass = await run_action(postgres_app.name, "get-password", username="operator") + db_pass = db_pass["password"] + return_code, _, scp_err = await ops_test.juju( + "scp", + "--container", + "postgresql", + "./testing_database/testing_database.sql", + f"{postgres_app.units[0].name}:.", + ) + + assert return_code == 0, scp_err + + return_code, _, ssh_err = await ops_test.juju( + "ssh", + "--container", + "postgresql", + postgres_app.units[0].name, + "createdb -h localhost -U operator --password discourse", + stdin=str.encode(f"{db_pass}\n"), + ) + assert return_code == 0, ssh_err + + return_code, _, ssh_err = await ops_test.juju( + "ssh", + "--container", + "postgresql", + postgres_app.units[0].name, + "pg_restore -h localhost -U operator\ + --password -d discourse\ + --no-owner --clean --if-exists ./testing_database.sql", + stdin=str.encode(f"{db_pass}\n"), + ) + assert return_code == 0, ssh_err + + # ensure we are using the Discourse v3.2.0 database + # Discourse v3.2.0 uses the git commit hash: + # f9502188a646cdb286ae6572ad6198c711ecdea8 + return_code, latest_git_version, _ = await ops_test.juju( + "ssh", + "--container", + "postgresql", + postgres_app.units[0].name, + "psql -h localhost -U operator\ + --password -d discourse\ + -c 'SELECT git_version FROM schema_migration_details LIMIT 1;'", + stdin=str.encode(f"{db_pass}\n"), + ) + assert ( + "f9502188a646cdb286ae6572ad6198c711ecdea8" in latest_git_version + ), "Discourse v3.2.0 git version does not match with the database version" + + redis_app = await model.deploy("redis-k8s", series="jammy", channel="latest/edge") + await model.wait_for_idle(apps=[redis_app.name], status="active") + + charm = await ops_test.build_charm(".") + await model.deploy("nginx-ingress-integrator", series="focal", trust=True) + app_name = "discourse-k8s" + discourse_application = await model.deploy( + charm, + resources={"discourse-image": pytestconfig.getoption("--discourse-image")}, + application_name=app_name, + series="focal", + ) + await model.wait_for_idle(apps=[app_name], status="waiting") + unit = discourse_application.units[0] + assert unit.workload_status == WaitingStatus.name # type: ignore + await model.add_relation(app_name, "postgresql-k8s:database") + await model.add_relation(app_name, "redis-k8s") + await model.add_relation(app_name, "nginx-ingress-integrator") + await model.wait_for_idle(apps=[app_name], status="active", raise_on_error=True) + await model.wait_for_idle(apps=[app_name], status="active", raise_on_error=True)