diff --git a/CHANGELOG.md b/CHANGELOG.md index 29771bbfe9e705b5c38c13d857802ae9240afc7a..b5a7ca29231ef0e63000639174e9be3c7eacae3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [fix] front: univ form was broken (updated backend serializer) - [feat] front / back: added stats view and frontend deps needed for the view - [feat] Create a custom admin site containing a view to run cron tasks manually. +- [feat] back: add functions and tests to compute daily stats about the contributions ## v2.6.0 diff --git a/backend/backend_app/tests/utils.py b/backend/backend_app/tests/utils.py index 5619afe932f9e6b350ab543e399a8a743515b4f1..240d88d0d6d310ca16245bcce0c74f4c9815bbf0 100644 --- a/backend/backend_app/tests/utils.py +++ b/backend/backend_app/tests/utils.py @@ -3,6 +3,7 @@ from django.test import TestCase from rest_framework.test import APIClient from backend_app.models.country import Country +from backend_app.models.partner import Partner from backend_app.models.university import University from base_app.models import User @@ -95,7 +96,7 @@ def get_dummy_country() -> Country: def get_dummy_university(id: int = 1) -> University: - return University.objects.update_or_create( + univ = University.objects.update_or_create( name=f"Université de Technologie de Compiègne_{id}", defaults=dict( acronym="UTC", @@ -105,3 +106,16 @@ def get_dummy_university(id: int = 1) -> University: city="", ), )[0] + # Also set up partner info, to complete the circle + partner = get_dummy_partner(id) + partner.university = univ + partner.save() + return univ + + +def get_dummy_partner(id: int = 1) -> Partner: + return Partner.objects.update_or_create( + univ_name=f"Université de Technologie de Compiègne_{id}", + utc_id=id, + defaults=dict(iso_code="ab", country="osef", city=""), + )[0] diff --git a/backend/base_app/admin.py b/backend/base_app/admin.py index e77eb93d7d49cc1bfaa9daf50541732eb2d066ae..f085a0619840f09a5084e2f4df75b95e50f710d8 100644 --- a/backend/base_app/admin.py +++ b/backend/base_app/admin.py @@ -15,6 +15,7 @@ class CustomAdminSite(admin.AdminSite): Custom admin site used to add a trigger_cron view on the admin site provided by django """ + def get_urls(self): urls = super().get_urls() urls += [path("trigger_cron/", self.admin_view(trigger_cron))] diff --git a/backend/stats_app/admin.py b/backend/stats_app/admin.py index ae9c8989317c50ea8c3dbdf935d65728380418e6..b84c48a97b23415acba1548c13fd759a05d16a3a 100644 --- a/backend/stats_app/admin.py +++ b/backend/stats_app/admin.py @@ -1,5 +1,6 @@ from base_app.admin import admin_site -from stats_app.models import DailyConnections +from stats_app.models import DailyConnections, DailyExchangeContributionsInfo admin_site.register(DailyConnections) +admin_site.register(DailyExchangeContributionsInfo) diff --git a/backend/stats_app/compute_stats.py b/backend/stats_app/compute_stats.py index 3e77f425dfcce882235e7191ca362b8b051da9d9..d285d432498d59e0132d0a602ea73f99eb7ff47b 100644 --- a/backend/stats_app/compute_stats.py +++ b/backend/stats_app/compute_stats.py @@ -1,7 +1,14 @@ from datetime import timedelta +from collections import Counter +from backend_app.models.university import University -from stats_app.models import DailyConnections -from stats_app.utils import get_daily_connections, get_today_as_datetime + +from stats_app.models import DailyConnections, DailyExchangeContributionsInfo +from stats_app.utils import ( + get_daily_connections, + get_today_as_datetime, + get_contributions_profiles, +) def update_daily_connections(): @@ -11,5 +18,24 @@ def update_daily_connections(): ) +def update_daily_exchange_contributions_info(): + yesterday = get_today_as_datetime() - timedelta(days=1) + + contributions_profiles = get_contributions_profiles() + + nb_contributions_by_profile = Counter(contributions_profiles) + for contribution_profile, nb_contributions in nb_contributions_by_profile.items(): + university = University.objects.get(pk=contribution_profile.university_pk) + DailyExchangeContributionsInfo.objects.update_or_create( + date=yesterday, + major=contribution_profile.major, + minor=contribution_profile.minor, + exchange_semester=contribution_profile.exchange_semester, + university=university, + defaults=dict(nb_contributions=nb_contributions), + ) + + def update_all_stats(): update_daily_connections() + update_daily_exchange_contributions_info() diff --git a/backend/stats_app/migrations/0002_auto_20200601_1418.py b/backend/stats_app/migrations/0002_auto_20200601_1418.py new file mode 100644 index 0000000000000000000000000000000000000000..654ff3e5acd7b62985f34d475b92e52c6e530b9d --- /dev/null +++ b/backend/stats_app/migrations/0002_auto_20200601_1418.py @@ -0,0 +1,53 @@ +# Generated by Django 2.1.7 on 2020-06-01 12:18 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("backend_app", "0005_lastvisiteduniversity"), + ("stats_app", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="DailyExchangeContributionsInfo", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date", models.DateTimeField()), + ("major", models.CharField(blank=True, max_length=20)), + ("minor", models.CharField(blank=True, max_length=47)), + ("exchange_semester", models.CharField(max_length=5)), + ( + "nb_contributions", + models.IntegerField( + validators=[django.core.validators.MinValueValidator(0)] + ), + ), + ( + "university", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="backend_app.University", + ), + ), + ], + ), + migrations.AlterUniqueTogether( + name="dailyexchangecontributionsinfo", + unique_together={ + ("date", "university", "major", "minor", "exchange_semester") + }, + ), + ] diff --git a/backend/stats_app/models.py b/backend/stats_app/models.py index 8f59831dec70edca7f9ae73ec5df794fcaec38de..f56ca5d8cbc5974f282963ec2ffce068d329e329 100644 --- a/backend/stats_app/models.py +++ b/backend/stats_app/models.py @@ -1,7 +1,22 @@ from django.db import models from django.core.validators import MinValueValidator +from backend_app.models.university import University class DailyConnections(models.Model): date = models.DateTimeField(unique=True, null=False) nb_connections = models.IntegerField(validators=[MinValueValidator(0)], null=False) + + +class DailyExchangeContributionsInfo(models.Model): + date = models.DateTimeField(null=False) + university = models.ForeignKey(University, null=False, on_delete=models.CASCADE) + major = models.CharField(max_length=20, null=False, blank=True) + minor = models.CharField(max_length=47, null=False, blank=True) + exchange_semester = models.CharField(max_length=5, null=False) + nb_contributions = models.IntegerField( + validators=[MinValueValidator(0)], null=False + ) + + class Meta: + unique_together = ("date", "university", "major", "minor", "exchange_semester") diff --git a/backend/stats_app/tests/test_nb_connections.py b/backend/stats_app/tests/test_nb_connections.py index 588664c03ce865210300b3d3a97c5d68bd0dbdf7..002baa17d7755b73498b6bec7f5e020195c3bf36 100644 --- a/backend/stats_app/tests/test_nb_connections.py +++ b/backend/stats_app/tests/test_nb_connections.py @@ -3,7 +3,7 @@ from datetime import timedelta from django.test import TestCase from base_app.models import User -from stats_app.compute_stats import update_all_stats +from stats_app.compute_stats import update_daily_connections from stats_app.models import DailyConnections from stats_app.utils import get_daily_connections, get_today_as_datetime @@ -29,7 +29,7 @@ class StatsConnectionsTest(TestCase): daily_connections = get_daily_connections() self.assertEqual(daily_connections, 10) - def test_update_all_stats(self): + def test_update_daily_connections(self): today = get_today_as_datetime() yesterday = today - timedelta(days=1) for i in range(10): @@ -37,7 +37,7 @@ class StatsConnectionsTest(TestCase): username=f"{i}", defaults=dict(last_login=yesterday) ) - update_all_stats() + update_daily_connections() yesterday_daily_connections = DailyConnections.objects.get(date=yesterday) diff --git a/backend/stats_app/tests/test_nb_contributions.py b/backend/stats_app/tests/test_nb_contributions.py new file mode 100644 index 0000000000000000000000000000000000000000..3c54e286c5b850e4b59773f8efdbb0241c2d2d6a --- /dev/null +++ b/backend/stats_app/tests/test_nb_contributions.py @@ -0,0 +1,69 @@ +from datetime import timedelta + +from django.test import TestCase + +from backend_app.models.exchangeFeedback import ExchangeFeedback +from backend_app.models.exchange import Exchange +from backend_app.tests.utils import get_dummy_university +from stats_app.compute_stats import update_daily_exchange_contributions_info +from stats_app.models import DailyExchangeContributionsInfo +from stats_app.utils import get_today_as_datetime, get_contributions_profiles + + +class StatsContributionsTest(TestCase): + @classmethod + def setUpTestData(cls): + today = get_today_as_datetime() + yesterday = today - timedelta(days=1) + + cls.univ = get_dummy_university() + utc_partner_id = cls.univ.corresponding_utc_partners.all()[0].utc_id + + for i in range(2): + exchange = Exchange.objects.update_or_create( + pk=i, + defaults=dict( + utc_id=i, + utc_partner_id=utc_partner_id, + year=2019, + semester="A", + student_major_and_semester="IM4", + student_minor="IDI", + duration=2, + double_degree=False, + master_obtained=False, + utc_allow_courses=True, + utc_allow_login=True, + university=cls.univ, + ), + )[0] + ExchangeFeedback.objects.update_or_create( + exchange=exchange, + defaults=dict( + updated_on=yesterday, untouched=False, university=cls.univ + ), + ) + + def test_get_contributions_profiles(self): + daily_contributions_profiles = get_contributions_profiles() + self.assertEqual(len(daily_contributions_profiles), 2) + for p in daily_contributions_profiles: + self.assertEqual(p.major, "IM") + self.assertEqual(p.minor, "IDI") + self.assertEqual(p.exchange_semester, "A2019") + self.assertEqual(p.university_pk, self.univ.pk) + + def test_update_daily_exchange_contributions_info(self): + today = get_today_as_datetime() + yesterday = today - timedelta(days=1) + update_daily_exchange_contributions_info() + + contributions = DailyExchangeContributionsInfo.objects.filter(date=yesterday) + + self.assertEqual(len(contributions), 1) + contribution: DailyExchangeContributionsInfo = contributions[0] + self.assertEqual(contribution.university.pk, self.univ.pk) + self.assertEqual(contribution.major, "IM") + self.assertEqual(contribution.minor, "IDI") + self.assertEqual(contribution.exchange_semester, "A2019") + self.assertEqual(contribution.nb_contributions, 2) diff --git a/backend/stats_app/utils.py b/backend/stats_app/utils.py index 1b9921a41503b57ad6073b0a2fba5f8f06555a10..321b2917af841a15be4df42b1e6423e53ad6fc62 100644 --- a/backend/stats_app/utils.py +++ b/backend/stats_app/utils.py @@ -1,8 +1,14 @@ from datetime import datetime, timedelta from django.utils.timezone import make_aware +from dataclasses import dataclass +from typing import List from base_app.models import User +from backend_app.models.exchangeFeedback import ExchangeFeedback +from backend_app.models.courseFeedback import CourseFeedback +from backend_app.models.exchange import Exchange + def get_today_as_datetime(): now = datetime.now() @@ -17,3 +23,60 @@ def get_daily_connections() -> int: last_login__gte=yesterday, last_login__lt=today ).count() return nb_connections + + +@dataclass(eq=True, frozen=True) +class ContributionProfile: + """ + Class for keeping track a profile. + """ + + major: str + minor: str + exchange_semester: str + university_pk: int + + +def _get_profile_from_exchange(exchange: Exchange) -> ContributionProfile: + return ContributionProfile( + major=exchange.student_major if exchange.student_major is not None else "", + minor=exchange.student_minor if exchange.student_minor is not None else "", + exchange_semester=f"{exchange.semester}{exchange.year}", + university_pk=exchange.university.pk, + ) + + +def get_contributions_profiles() -> List[ContributionProfile]: + """ + return the yesterday contributions profiles + + If no university if associated with a contribution we don't return it + """ + today = get_today_as_datetime() + yesterday = today - timedelta(days=1) + contributions_profiles = [] + + exchange_feedbacks = ExchangeFeedback.objects.filter( + updated_on__gte=yesterday, + updated_on__lt=today, + exchange__university__isnull=False, + untouched=False, + ).prefetch_related("exchange") + + for exchange_feedback in exchange_feedbacks: + exchange = exchange_feedback.exchange + contribution_profile = _get_profile_from_exchange(exchange) + contributions_profiles.append(contribution_profile) + + course_feedbacks = CourseFeedback.objects.filter( + updated_on__gte=yesterday, + updated_on__lt=today, + course__exchange__university__isnull=False, + untouched=False, + ).prefetch_related("course__exchange") + for course_feedback in course_feedbacks: + exchange = course_feedback.course.exchange + contribution_profile = _get_profile_from_exchange(exchange) + contributions_profiles.append(contribution_profile) + + return contributions_profiles