Feat: Fixed db migration issues from v3.2.0 to 3.3.0 (#287)

* Feat: Fixed db migration issues from v3.2.0 to 3.3.0

* chore: Fix charmcraft pack

* chore: Fix patch

* chore: Tried to create the mock database with api calls but failed

* chore: Finalized integration tests for db migration

* chore: Fix postgres revision in test

* chore: Delete unnecessary line, more explanation in test docstring

* chore: Move test_db_migration to its own module and add it into git workflow

* chore: Format

* chore: Add documentation for db creation and rename the testing db

* chore: Fix typo and delete unnecessary mock_db

* chore: Ensured test db version, updated documentation

* chore: Removed extra empty space

.github/workflows/integration_test.yaml

@@ -13,6 +13,6 @@ jobs:
       trivy-image-config: "trivy.yaml"
       juju-channel: 3.1/stable
       channel: 1.28-strict/stable
-      modules: '["test_charm", "test_saml", "test_users"]'
+      modules: '["test_charm", "test_saml", "test_users", "test_db_migration"]'
       self-hosted-runner: true
       self-hosted-runner-label: "edge"

charmcraft.yaml

@@ -12,5 +12,6 @@ bases:
       channel: "20.04"
 parts:
   charm:
+    build-packages: [cargo, rustc, pkg-config, libffi-dev, libssl-dev]
     charm-python-packages: [setuptools, pip]  # https://discourse.charmhub.io/t/install-or-update-python-packages-before-packing-a-charm/5158
     charm-binary-python-packages: [cosl]  # https://github.com/canonical/charmcraft/issues/1269

discourse_rock/patches/db_migrations.patch

@@ -0,0 +1,16 @@
+diff --git a/db/post_migrate/20240212034010_drop_deprecated_columns.rb b/db/post_migrate/20240212034010_drop_deprecated_columns.rb
+index 0899da20..015fc6d5 100644
+--- a/db/post_migrate/20240212034010_drop_deprecated_columns.rb
++++ b/db/post_migrate/20240212034010_drop_deprecated_columns.rb
+@@ -19,6 +19,11 @@ class DropDeprecatedColumns < ActiveRecord::Migration[7.0]
+   }
+
+   def up
++    execute <<~SQL
++      DROP TRIGGER IF EXISTS invites_user_id_readonly ON invites;
++      DROP TRIGGER IF EXISTS invites_redeemed_at_readonly ON invites;
++      DROP TRIGGER IF EXISTS user_api_keys_scopes_readonly ON user_api_keys;
++    SQL
+     DROPPED_COLUMNS.each { |table, columns| Migration::ColumnDropper.execute_drop(table, columns) }
+   end
+

discourse_rock/rockcraft.yaml

@@ -192,6 +192,7 @@ parts:
       - git
     after: [discourse, patches]
     override-stage: |
+      git -C srv/discourse/app apply patches/db_migrations.patch
       git -C srv/discourse/app apply patches/lp1903695.patch
       git -C srv/discourse/app apply patches/discourse-charm.patch
       git -C srv/discourse/app apply patches/sigterm.patch

testing_database/creating-the-testing-database.md

@@ -0,0 +1,35 @@
+# Create the testing database
+At the writing this document, the testing database is created using Discourse v3.2.0.
+To get the same result use Charm revision 162 and resource revision 152.
+
+## Create the database
+First of all we need to deploy the Discourse following the [tutorial](https://github.com/canonical/discourse-k8s-operator/blob/main/docs/tutorial.md).
+
+Then, we need to create a 2 new users for testing using the actions:
+
+```juju run discourse-k8s/0 create-user [email protected] admin=true```
+```juju run discourse-k8s/0 create-user [email protected]```
+
+Please note that the first user is an admin and the second one is not. Also please not the passwords that are generated automatically by the command.
+
+Now open the Discourse URL in a browser, login with the first user (admin) and create a new topic. Reply to this topic as the admin user again. Then, login with the second user and reply to this topic. Then login with the first user and approve the second users reply.
+
+## Export the database
+
+First we need to get the database password:
+```juju run postgresql-k8s/0 get-password username=operator```
+
+Ssh into the database
+```juju ssh --container postgresql postgresql-k8s/0 bash```
+
+Create a folder to dump the db
+```mkdir -p /srv/dump/```
+
+Dump the db. Ip here is the unit ip
+```pg_dump -Fc -h 10.1.187.134 -U operator -d discourse > "/srv/dump/testing_database.sql"```
+
+Exit the container
+```exit```
+
+Copy the dump into local file system.
+```juju scp --container postgresql postgresql-k8s/0:/srv/dump/testing_database.sql./testing_database.sql```

tests/integration/test_db_migration.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Discourse integration tests."""
+
+import logging
+
+import pytest
+from botocore.config import Config
+from ops.model import WaitingStatus
+from pytest_operator.plugin import Model, OpsTest
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.asyncio
+@pytest.mark.abort_on_fail
+async def test_db_migration(model: Model, ops_test: OpsTest, pytestconfig: Config, run_action):
+    """
+    arrange: preload postgres with a testing db that is created in Discourse v3.2.0
+    act: deploy and integrate with Discourse v3.3.0 (latest)
+    assert: discourse is active/idle
+
+    Discourse must be active idle, it might create migration errors related to
+    not being able to delete some columns because of triggers. This is fixed
+    with a patch but this patch only works for Discourse v3.2.0 and we might
+    need to create a new patch for the new version of Discourse.
+    """
+    postgres_app = await model.deploy(
+        "postgresql-k8s",
+        channel="14/stable",
+        series="jammy",
+        revision=300,
+        trust=True,
+        config={"profile": "testing"},
+    )
+    async with ops_test.fast_forward():
+        await model.wait_for_idle(apps=[postgres_app.name], status="active")
+    await postgres_app.set_config(
+        {
+            "plugin_hstore_enable": "true",
+            "plugin_pg_trgm_enable": "true",
+        }
+    )
+    await model.wait_for_idle(apps=[postgres_app.name], status="active")
+    db_pass = await run_action(postgres_app.name, "get-password", username="operator")
+    db_pass = db_pass["password"]
+    return_code, _, scp_err = await ops_test.juju(
+        "scp",
+        "--container",
+        "postgresql",
+        "./testing_database/testing_database.sql",
+        f"{postgres_app.units[0].name}:.",
+    )
+
+    assert return_code == 0, scp_err
+
+    return_code, _, ssh_err = await ops_test.juju(
+        "ssh",
+        "--container",
+        "postgresql",
+        postgres_app.units[0].name,
+        "createdb -h localhost -U operator --password discourse",
+        stdin=str.encode(f"{db_pass}\n"),
+    )
+    assert return_code == 0, ssh_err
+
+    return_code, _, ssh_err = await ops_test.juju(
+        "ssh",
+        "--container",
+        "postgresql",
+        postgres_app.units[0].name,
+        "pg_restore -h localhost -U operator\
+              --password -d discourse\
+                  --no-owner --clean --if-exists ./testing_database.sql",
+        stdin=str.encode(f"{db_pass}\n"),
+    )
+    assert return_code == 0, ssh_err
+
+    # ensure we are using the Discourse v3.2.0 database
+    # Discourse v3.2.0 uses the git commit hash:
+    # f9502188a646cdb286ae6572ad6198c711ecdea8
+    return_code, latest_git_version, _ = await ops_test.juju(
+        "ssh",
+        "--container",
+        "postgresql",
+        postgres_app.units[0].name,
+        "psql -h localhost -U operator\
+              --password -d discourse\
+                  -c 'SELECT git_version FROM schema_migration_details LIMIT 1;'",
+        stdin=str.encode(f"{db_pass}\n"),
+    )
+    assert (
+        "f9502188a646cdb286ae6572ad6198c711ecdea8" in latest_git_version
+    ), "Discourse v3.2.0 git version does not match with the database version"
+
+    redis_app = await model.deploy("redis-k8s", series="jammy", channel="latest/edge")
+    await model.wait_for_idle(apps=[redis_app.name], status="active")
+
+    charm = await ops_test.build_charm(".")
+    await model.deploy("nginx-ingress-integrator", series="focal", trust=True)
+    app_name = "discourse-k8s"
+    discourse_application = await model.deploy(
+        charm,
+        resources={"discourse-image": pytestconfig.getoption("--discourse-image")},
+        application_name=app_name,
+        series="focal",
+    )
+    await model.wait_for_idle(apps=[app_name], status="waiting")
+    unit = discourse_application.units[0]
+    assert unit.workload_status == WaitingStatus.name  # type: ignore
+    await model.add_relation(app_name, "postgresql-k8s:database")
+    await model.add_relation(app_name, "redis-k8s")
+    await model.add_relation(app_name, "nginx-ingress-integrator")
+    await model.wait_for_idle(apps=[app_name], status="active", raise_on_error=True)
+    await model.wait_for_idle(apps=[app_name], status="active", raise_on_error=True)