feat(stats): collecter les stats d'activité hebdo des forums de la do…

…cumentation (#691)

## Description

🎸 Interroger matomo pour collecter le nombre de visites uniques, le
nombre de visites uniques entrantes et le temps passé sur les `forum` de
l'espace documentation hebdomadairement
🎸 Management command planifiée chaque lundi
🎸 Capacité à reprendre un traitement échoué, en recherchant la plus
récente date d'execution dans `ForumStat`

## Type de changement

🎢 Nouvelle fonctionnalité (changement non cassant qui ajoute une
fonctionnalité).

### Points d'attention

🦺 Collecte limitée aux `forum` de l'espace documentation
🦺 La collecte des données mensuelles est prévue dans une PR ultérieure
🦺 harmonisation de la déclaration du modèle `Stat` dans le fichier
`admin.py`

clevercloud/collect_weekly_matomo_forum_stats.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -l
+
+# Collect Daily Matomo Stats
+
+#
+# About clever cloud cronjobs:
+# https://www.clever-cloud.com/doc/tools/crons/
+#
+
+if [[ "$INSTANCE_NUMBER" != "0" ]]; then
+    echo "Instance number is ${INSTANCE_NUMBER}. Stop here."
+    exit 0
+fi
+
+# $APP_HOME is set by default by clever cloud.
+cd $APP_HOME
+
+python manage.py collect_matomo_forum_stats

clevercloud/cron.json

@@ -4,6 +4,7 @@
     "0 5 * * * $ROOT/clevercloud/collect_daily_matomo_stats.sh",
     "3 5 * * * $ROOT/clevercloud/collect_daily_django_stats.sh",
     "5 5 1 * * $ROOT/clevercloud/collect_monthly_matomo_stats.sh",
+    "8 5 * * 1 $ROOT/clevercloud/collect_weekly_matomo_forum_stats.sh",
     "5 7-21 * * * $ROOT/clevercloud/send_notifs_when_first_reply.sh",
     "5 6 * * * $ROOT/clevercloud/send_notifs_when_following_replies.sh",
     "10 6-22 * * * $ROOT/clevercloud/add_user_to_list_when_register.sh",

lacommunaute/stats/admin.py

@@ -1,11 +1,16 @@
 from django.contrib import admin
 
-from lacommunaute.stats.models import Stat
+from lacommunaute.stats.models import ForumStat, Stat
 
 
+@admin.register(Stat)
 class StatAdmin(admin.ModelAdmin):
     list_display = ("name", "date", "value", "period")
     list_filter = ("name", "date", "period")
 
 
-admin.site.register(Stat, StatAdmin)
+@admin.register(ForumStat)
+class ForumStatAdmin(admin.ModelAdmin):
+    list_display = ("date", "period", "forum", "visits", "entry_visits", "time_spent")
+    list_filter = ("date", "period", "forum")
+    raw_id_fields = ("forum",)

lacommunaute/stats/factories.py

@@ -3,8 +3,9 @@
 import factory
 import factory.django
 
+from lacommunaute.forum.factories import ForumFactory
 from lacommunaute.stats.enums import Period
-from lacommunaute.stats.models import Stat
+from lacommunaute.stats.models import ForumStat, Stat
 
 
 class StatFactory(factory.django.DjangoModelFactory):
@@ -23,3 +24,15 @@ class Params:
             value=46,
             period="day",
         )
+
+
+class ForumStatFactory(factory.django.DjangoModelFactory):
+    date = factory.Faker("date")
+    period = Period.DAY
+    forum = factory.SubFactory(ForumFactory)
+    visits = factory.Faker("pyint")
+    entry_visits = factory.Faker("pyint")
+    time_spent = factory.Faker("pyint")
+
+    class Meta:
+        model = ForumStat

lacommunaute/stats/management/commands/collect_matomo_forum_stats.py

@@ -0,0 +1,28 @@
+from datetime import date
+
+from dateutil.relativedelta import relativedelta
+from django.core.management.base import BaseCommand
+
+from lacommunaute.stats.models import ForumStat
+from lacommunaute.utils.date import get_last_sunday
+from lacommunaute.utils.matomo import collect_forum_stats_from_matomo_api
+
+
+class Command(BaseCommand):
+    help = "Collecter les stats des forum dans matomo, jusqu'au dimanche précédent l'execution"
+
+    def handle(self, *args, **options):
+        period = "week"
+
+        from_date = ForumStat.objects.filter(period=period).order_by("-date").first()
+
+        if from_date:
+            from_date = from_date.date + relativedelta(days=7)
+        else:
+            from_date = date(2023, 10, 2)
+
+        to_date = get_last_sunday(date.today())
+
+        collect_forum_stats_from_matomo_api(from_date=from_date, to_date=to_date, period=period)
+
+        self.stdout.write(self.style.SUCCESS("That's all, folks!"))

lacommunaute/stats/migrations/0002_forumstat.py

@@ -0,0 +1,44 @@
+# Generated by Django 5.0.6 on 2024-06-24 15:02
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("forum", "0015_alter_forumrating_options"),
+        ("stats", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ForumStat",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("date", models.DateField(verbose_name="Date")),
+                (
+                    "period",
+                    models.CharField(
+                        choices=[("month", "Month"), ("week", "Week"), ("day", "Day")],
+                        max_length=10,
+                        verbose_name="Période",
+                    ),
+                ),
+                ("visits", models.IntegerField(default=0, verbose_name="Visites")),
+                ("entry_visits", models.IntegerField(default=0, verbose_name="Visites entrantes")),
+                ("time_spent", models.IntegerField(default=0, verbose_name="Temps passé")),
+                (
+                    "forum",
+                    models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.SET_NULL, to="forum.forum", verbose_name="Forum"
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Stat de forum",
+                "verbose_name_plural": "Stats de forum",
+                "ordering": ["date", "period", "forum"],
+                "unique_together": {("date", "period", "forum")},
+            },
+        ),
+    ]

lacommunaute/stats/models.py

@@ -1,5 +1,6 @@
 from django.db import models
 
+from lacommunaute.forum.models import Forum
 from lacommunaute.stats.enums import Period
 
 
@@ -14,6 +15,10 @@ def current_month_datas(self):
 
 
 class Stat(models.Model):
+    """
+    Represents a statistical data point, relative to the whole platform, for a given date and period.
+    """
+
     name = models.CharField(max_length=30, verbose_name="Nom")
     date = models.DateField(verbose_name="Date")
     value = models.IntegerField(verbose_name="Valeur")
@@ -31,3 +36,27 @@ def __str__(self):
         return f"{self.name} - {self.date} - {self.period}"
 
     objects = StatQuerySet().as_manager()
+
+
+class ForumStat(models.Model):
+    """
+    Represents a statistical data point, relative to a forum, for a given date and period.
+    """
+
+    date = models.DateField(verbose_name="Date")
+    period = models.CharField(max_length=10, verbose_name="Période", choices=Period.choices)
+    forum = models.ForeignKey(Forum, on_delete=models.SET_NULL, verbose_name="Forum", null=True)
+    visits = models.IntegerField(verbose_name="Visites", default=0)
+    entry_visits = models.IntegerField(verbose_name="Visites entrantes", default=0)
+    time_spent = models.IntegerField(verbose_name="Temps passé", default=0)
+
+    objects = models.Manager()
+
+    class Meta:
+        verbose_name = "Stat de forum"
+        verbose_name_plural = "Stats de forum"
+        ordering = ["date", "period", "forum"]
+        unique_together = ("date", "period", "forum")
+
+    def __str__(self):
+        return f"{self.date} - {self.period} - {self.forum}"

lacommunaute/stats/tests/tests_models.py

@@ -1,12 +1,13 @@
+import pytest  # noqa
 from dateutil.relativedelta import relativedelta
 from django.db import IntegrityError
 from django.test import TestCase
 from django.utils import timezone
 from django.utils.timezone import localdate
 
 from lacommunaute.stats.enums import Period
-from lacommunaute.stats.factories import StatFactory
-from lacommunaute.stats.models import Stat
+from lacommunaute.stats.factories import ForumStatFactory, StatFactory
+from lacommunaute.stats.models import ForumStat, Stat
 
 
 class StatModelTest(TestCase):
@@ -31,3 +32,21 @@ def test_ordering(self):
 
     def test_empty_dataset(self):
         self.assertEqual(Stat.objects.current_month_datas().count(), 0)
+
+
+class TestForumStat:
+    def test_ordering(self, db):
+        first_forumstat = ForumStatFactory(date=localdate())
+        second_forumstat = ForumStatFactory(
+            forum=first_forumstat.forum,
+            date=first_forumstat.date + relativedelta(days=1),
+            period=first_forumstat.period,
+        )
+
+        assert list(ForumStat.objects.all()) == [first_forumstat, second_forumstat]
+
+    def test_uniqueness(self, db):
+        forumstat = ForumStatFactory()
+        forumstat.id = None
+        with pytest.raises(IntegrityError):
+            forumstat.save()

lacommunaute/utils/date.py

@@ -0,0 +1,8 @@
+from datetime import date
+
+from dateutil.relativedelta import relativedelta
+
+
+def get_last_sunday(theday=date.today()):
+    days_to_subtract = (theday.weekday() + 1) % 7
+    return theday - relativedelta(days=days_to_subtract)

lacommunaute/utils/matomo.py

@@ -5,7 +5,8 @@
 from dateutil.relativedelta import relativedelta
 from django.conf import settings
 
-from lacommunaute.stats.models import Stat
+from lacommunaute.forum.models import Forum
+from lacommunaute.stats.models import ForumStat, Stat
 
 
 def get_matomo_data(
@@ -138,6 +139,42 @@ def get_matomo_events_data(period, search_date, nb_uniq_visitors_key="nb_uniq_vi
     return stats
 
 
+def get_matomo_forums_data(period, search_date, label, ids=[]):
+    if label is None:
+        raise ValueError("label must be provided")
+
+    filtered_datas = next(
+        (
+            data.get("subtable", [])
+            for data in get_matomo_data(period=period, search_date=search_date, method="Actions.getPageUrls")
+            if data.get("label") == label
+        ),
+        [],
+    )
+
+    stats = {}
+    for forum_data in filtered_datas:
+        forum_id = int(forum_data["label"].split("-")[-1]) if forum_data["label"].split("-")[-1].isdigit() else None
+
+        if forum_id and forum_id in ids:
+            # ONE forum can have multiple slugs. We need to aggregate them.
+            stats.setdefault(
+                forum_id,
+                {
+                    "date": search_date.strftime("%Y-%m-%d"),
+                    "period": period,
+                    "visits": 0,
+                    "entry_visits": 0,
+                    "time_spent": 0,
+                },
+            )
+            stats[forum_id]["visits"] += forum_data.get("nb_visits", 0)
+            stats[forum_id]["entry_visits"] += forum_data.get("entry_nb_visits", 0)
+            stats[forum_id]["time_spent"] += forum_data.get("sum_time_spent", 0)
+
+    return [{"forum_id": k, **v} for k, v in stats.items()]
+
+
 def collect_stats_from_matomo_api(period="day", from_date=date(2022, 12, 5), to_date=date.today()):
     """
     function to get stats from matomo api, day by day from 2022-10-31 to today
@@ -157,3 +194,34 @@ def collect_stats_from_matomo_api(period="day", from_date=date(2022, 12, 5), to_
             from_date += relativedelta(months=1)
 
     Stat.objects.bulk_create([Stat(**stat) for stat in stats])
+
+
+def collect_forum_stats_from_matomo_api(period="week", from_date=date(2023, 10, 2), to_date=date.today()):
+    if period != "week":
+        raise ValueError("Only 'week' period is supported for forum stats collection.")
+
+    forums_dict = {
+        forum.id: forum
+        for forum in Forum.objects.filter(parent__type=Forum.FORUM_CAT, level=1)
+        | Forum.objects.filter(type=Forum.FORUM_CAT, level=0)
+    }
+
+    search_date = from_date
+    while search_date <= to_date:
+        forums_stats = get_matomo_forums_data(period, search_date, label="forum", ids=list(forums_dict.keys()))
+        print(f"Stats collected for {period} {search_date} ({len(forums_stats)} stats collected)")
+
+        forum_stats_objects = [
+            {
+                "date": stat["date"],
+                "period": stat["period"],
+                "forum": forums_dict[stat["forum_id"]],
+                "visits": stat["visits"],
+                "entry_visits": stat["entry_visits"],
+                "time_spent": stat["time_spent"],
+            }
+            for stat in forums_stats
+        ]
+        ForumStat.objects.bulk_create([ForumStat(**stat) for stat in forum_stats_objects])
+
+        search_date += relativedelta(days=7)