[PILOTAGE] extraction des données Forum (part 1)

clevercloud/cron.json

@@ -1,4 +1,5 @@
 [
+    "0 3 * * 0 $ROOT/clevercloud/extract_tables.sh",
     "0 4 * * 0 $ROOT/clevercloud/rebuild_index.sh",
     "5 4 * * * $ROOT/clevercloud/update_index_daily.sh",
     "0 5 * * * $ROOT/clevercloud/collect_daily_matomo_stats.sh",

clevercloud/extract_tables.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -l
+
+# Rebuild index for the search engine.
+#
+# About clever cloud cronjobs:
+# https://www.clever-cloud.com/doc/tools/crons/
+#
+
+if [[ "$INSTANCE_NUMBER" != "0" ]]; then
+    echo "Instance number is ${INSTANCE_NUMBER}. Stop here."
+    exit 0
+fi
+
+# $APP_HOME is set by default by clever cloud.
+cd $APP_HOME
+
+python manage.py extract_tables

config/settings/base.py

@@ -80,6 +80,7 @@
     "lacommunaute.inclusion_connect",
     "lacommunaute.pages",
     "lacommunaute.forum_file",
+    "lacommunaute.metabase",
 ]
 
 INSTALLED_APPS = DJANGO_APPS + LOCAL_APPS + THIRD_PARTIES_APPS

lacommunaute/metabase/admin.py

@@ -0,0 +1,23 @@
+from django.contrib import admin
+
+from lacommunaute.metabase.models import ForumTable
+
+
+@admin.register(ForumTable)
+class ForumTableAdmin(admin.ModelAdmin):
+    readonly_fields = (
+        "name",
+        "kind",
+        "type",
+        "short_description_boolean",
+        "description_boolean",
+        "parent_name",
+        "direct_topics_count",
+        "upvotes_count",
+        "last_post_at",
+        "last_updated_at",
+        "extracted_at",
+    )
+
+    def has_add_permission(self, request):
+        return False

lacommunaute/metabase/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class MetabaseAppConfig(AppConfig):
+    label = "metabase"
+    name = "lacommunaute.metabase"
+    verbose_name = "Metabase"
+    verbose_name_plural = "Metabase"

lacommunaute/metabase/factories.py

@@ -0,0 +1,8 @@
+import factory
+
+from lacommunaute.metabase.models import ForumTable
+
+
+class ForumTableFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = ForumTable

lacommunaute/metabase/management/commands/extract_tables.py

@@ -0,0 +1,54 @@
+from datetime import datetime
+
+from django.core.management.base import BaseCommand
+from django.db.models import BooleanField, Count, ExpressionWrapper, Q
+
+from lacommunaute.forum.models import Forum
+from lacommunaute.metabase.models import ForumTable
+
+
+class Command(BaseCommand):
+    help = "Extracts data for the metabase"
+
+    def truncate_forum_tables(self):
+        ForumTable.objects.all().delete()
+
+    def extract_forum_tables(self, extracted_at):
+        TYPE_CHOICES_DICT = {item[0]: item[1] for item in Forum.TYPE_CHOICES}
+        forum_tables = []
+        for forum in (
+            Forum.objects.annotate(
+                short_description_boolean=ExpressionWrapper(
+                    Q(short_description__isnull=False), output_field=BooleanField()
+                )
+            )
+            .annotate(description_boolean=ExpressionWrapper(Q(description__isnull=False), output_field=BooleanField()))
+            .annotate(upvotes_count=Count("upvotes"))
+            .select_related("parent")
+        ):
+            forum_tables.append(
+                ForumTable(
+                    name=forum.name,
+                    kind=forum.kind,
+                    type=TYPE_CHOICES_DICT.get(forum.type),
+                    short_description_boolean=forum.short_description_boolean,
+                    description_boolean=forum.description_boolean,
+                    parent_name=forum.parent.name if forum.parent else None,
+                    direct_topics_count=forum.direct_topics_count,
+                    upvotes_count=forum.upvotes_count,
+                    last_post_at=forum.last_post_on,
+                    last_updated_at=forum.created,
+                    extracted_at=extracted_at,
+                )
+            )
+            self.stdout.write(self.style.SUCCESS(f"Extracted {forum.name}"))
+
+        ForumTable.objects.bulk_create(forum_tables)
+
+    def handle(self, *args, **options):
+        extracted_at = datetime.now()
+
+        self.truncate_forum_tables()
+        self.extract_forum_tables(extracted_at)
+
+        self.stdout.write(self.style.SUCCESS("That's all, folks!"))

lacommunaute/metabase/migrations/0001_initial.py

@@ -0,0 +1,55 @@
+# Generated by Django 4.2.7 on 2023-12-05 15:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="ForumTable",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("name", models.CharField(max_length=100, verbose_name="Nom")),
+                (
+                    "kind",
+                    models.CharField(
+                        choices=[
+                            ("PUBLIC_FORUM", "Espace public"),
+                            ("PRIVATE_FORUM", "Espace privé"),
+                            ("NEWS", "Actualités"),
+                        ],
+                        default="PUBLIC_FORUM",
+                        max_length=20,
+                        verbose_name="Kind",
+                    ),
+                ),
+                ("type", models.CharField(max_length=20, verbose_name="Type")),
+                (
+                    "short_description_boolean",
+                    models.BooleanField(default=False, verbose_name="Présence d'une description courte"),
+                ),
+                ("description_boolean", models.BooleanField(default=False, verbose_name="Présence d'une description")),
+                ("parent_name", models.CharField(max_length=100, null=True, verbose_name="Nom du forum parent")),
+                (
+                    "direct_topics_count",
+                    models.PositiveIntegerField(default=0, verbose_name="Nombre de sujets directs"),
+                ),
+                ("upvotes_count", models.PositiveIntegerField(default=0, verbose_name="Nombre d'upvotes")),
+                ("last_post_at", models.DateTimeField(null=True, verbose_name="Date du dernier message")),
+                ("last_updated_at", models.DateTimeField(null=True, verbose_name="Date de dernière mise à jour")),
+                (
+                    "extracted_at",
+                    models.DateTimeField(auto_now_add=True, verbose_name="Date d'extraction des données"),
+                ),
+            ],
+            options={
+                "verbose_name": "Forum Table",
+                "verbose_name_plural": "Forums Table",
+                "ordering": ["name"],
+            },
+        ),
+    ]

lacommunaute/metabase/models.py

@@ -0,0 +1,29 @@
+from django.db import models
+
+from lacommunaute.forum.enums import Kind as Forum_Kind
+
+
+class ForumTable(models.Model):
+    name = models.CharField(max_length=100, verbose_name="Nom")
+    kind = models.CharField(
+        max_length=20, choices=Forum_Kind.choices, default=Forum_Kind.PUBLIC_FORUM, verbose_name="Kind"
+    )
+    type = models.CharField(max_length=20, verbose_name="Type")
+    short_description_boolean = models.BooleanField(default=False, verbose_name="Présence d'une description courte")
+    description_boolean = models.BooleanField(default=False, verbose_name="Présence d'une description")
+    parent_name = models.CharField(null=True, max_length=100, verbose_name="Nom du forum parent")
+    direct_topics_count = models.PositiveIntegerField(default=0, verbose_name="Nombre de sujets directs")
+    upvotes_count = models.PositiveIntegerField(default=0, verbose_name="Nombre d'upvotes")
+    last_post_at = models.DateTimeField(null=True, verbose_name="Date du dernier message")
+    last_updated_at = models.DateTimeField(null=True, verbose_name="Date de dernière mise à jour")
+    extracted_at = models.DateTimeField(auto_now_add=True, verbose_name="Date d'extraction des données")
+
+    objects = models.Manager()
+
+    class Meta:
+        verbose_name = "Forum Table"
+        verbose_name_plural = "Forums Table"
+        ordering = ["name"]
+
+    def __str__(self):
+        return self.name

lacommunaute/metabase/tests/test_admin.py

@@ -0,0 +1,19 @@
+import pytest  # noqa: F401
+from django.urls import reverse
+from pytest_django.asserts import assertNotContains
+
+from lacommunaute.metabase.factories import ForumTableFactory
+
+
+@pytest.mark.django_db
+def test_add_button_hidden_on_admin_page_for_forumtable(admin_client):
+    response = admin_client.get(reverse("admin:metabase_forumtable_changelist"))
+    assertNotContains(response, reverse("admin:metabase_forumtable_add"), status_code=200)
+
+
+@pytest.mark.django_db
+def test_fields_are_readonly(admin_client):
+    forumtable = ForumTableFactory()
+    response = admin_client.get(reverse("admin:metabase_forumtable_change", kwargs={"object_id": forumtable.pk}))
+    assert response.status_code == 200
+    assert not response.context_data["adminform"].form.fields

lacommunaute/metabase/tests/test_extract_tables_command.py

@@ -0,0 +1,46 @@
+import pytest  # noqa: F401
+from django.core.management import call_command
+
+from lacommunaute.forum.factories import CategoryForumFactory, ForumFactory
+from lacommunaute.forum_conversation.factories import TopicFactory
+from lacommunaute.metabase.factories import ForumTableFactory
+from lacommunaute.metabase.models import ForumTable
+from lacommunaute.users.factories import UserFactory
+
+
+@pytest.mark.django_db
+def test_extract_forum_tables_command():
+    upvoted_forum = ForumFactory(upvoted_by=[UserFactory()])
+    TopicFactory(forum=upvoted_forum, with_post=True)
+
+    category_forum_with_child = CategoryForumFactory(with_child=True, description=None, short_description=None)
+
+    call_command("extract_tables")
+
+    assert ForumTable.objects.count() == 3
+    assert ForumTable.objects.filter(name=upvoted_forum.name).exists()
+    assert ForumTable.objects.filter(name=category_forum_with_child.name).exists()
+    assert ForumTable.objects.filter(name=category_forum_with_child.get_children().first().name).exists()
+
+    upvoted_forum_table = ForumTable.objects.get(name=upvoted_forum.name)
+    assert upvoted_forum_table.short_description_boolean
+    assert upvoted_forum_table.description_boolean
+    assert upvoted_forum_table.upvotes_count == 1
+    assert upvoted_forum_table.parent_name is None
+
+    category_forum_table = ForumTable.objects.get(name=category_forum_with_child.name)
+    assert category_forum_table.short_description_boolean is False
+    assert category_forum_table.description_boolean is False
+    assert category_forum_table.upvotes_count == 0
+    assert category_forum_table.parent_name is None
+
+    child_forum_table = ForumTable.objects.get(name=category_forum_with_child.get_children().first().name)
+    assert child_forum_table.parent_name == category_forum_with_child.name
+
+
+@pytest.mark.django_db
+def test_truncate_forum_table():
+    ForumTableFactory()
+    assert ForumTable.objects.count() == 1
+    call_command("extract_tables")
+    assert ForumTable.objects.count() == 0