Merge branch 'performance/re_setup_analysis' into 'master'

Performance optimization

Closes #55 and #54

See merge request !65

backend/backend_app/load_data/loading_scripts/loadAdminUser.py

-from django.contrib.auth.models import User
+from base_app.models import User
 import os
 
 

backend/backend_app/load_data/loading_scripts/loadCountries.py

 import os
 
-from django.contrib.auth.models import User
+from base_app.models import User
 
 import pandas as pd
 from backend_app.models.country import Country

backend/backend_app/load_data/loading_scripts/loadCurrencies.py

@@ -2,7 +2,7 @@ import csv
 import os
 from decimal import Decimal
 
-from django.contrib.auth.models import User
+from base_app.models import User
 
 from backend_app.models.currency import Currency
 

backend/backend_app/load_data/loading_scripts/loadGeneric.py

-from django.contrib.auth.models import User
+from base_app.models import User
 from django.utils import timezone
 
 import reversion

backend/backend_app/load_data/loading_scripts/loadTags.py

 import json
 import os
 
-from django.contrib.auth.models import User
+from base_app.models import User
 
 from backend_app.models.tag import Tag
 

backend/backend_app/load_data/loading_scripts/loadUniversities.py

 import os
 
-from django.contrib.auth.models import User
+from base_app.models import User
 
 import pandas as pd
 from backend_app.models.campus import Campus

backend/backend_app/load_data/loading_scripts/loadUniversityEx.py

 from datetime import datetime
 
-from django.contrib.auth.models import User
+from base_app.models import User
 
 from backend_app.models.country import Country, CountryScholarship
 from backend_app.models.currency import Currency

backend/backend_app/migrations/0002_auto_20180916_1555.py

-# Generated by Django 2.0.3 on 2018-09-16 13:55
-
-import backend_app.models.university.university
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [("backend_app", "0001_initial")]
-
-    operations = [
-        migrations.AlterField(
-            model_name="campus",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="campus",
-            name="name",
-            field=models.CharField(blank=True, default="", max_length=200),
-        ),
-        migrations.AlterField(
-            model_name="campus",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="campustaggeditem",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="campustaggeditem",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="city",
-            name="area",
-            field=models.CharField(blank=True, default="", max_length=200),
-        ),
-        migrations.AlterField(
-            model_name="city",
-            name="local_name",
-            field=models.CharField(blank=True, default="", max_length=200),
-        ),
-        migrations.AlterField(
-            model_name="citytaggeditem",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="citytaggeditem",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="country",
-            name="intermediate_region_name",
-            field=models.CharField(blank=True, default="", max_length=200),
-        ),
-        migrations.AlterField(
-            model_name="country",
-            name="intermediate_region_un_code",
-            field=models.CharField(blank=True, default="", max_length=3),
-        ),
-        migrations.AlterField(
-            model_name="country",
-            name="iso_alpha3_code",
-            field=models.CharField(blank=True, default="", max_length=3, unique=True),
-        ),
-        migrations.AlterField(
-            model_name="country",
-            name="sub_region_name",
-            field=models.CharField(blank=True, default="", max_length=200),
-        ),
-        migrations.AlterField(
-            model_name="country",
-            name="sub_region_un_code",
-            field=models.CharField(blank=True, default="", max_length=3),
-        ),
-        migrations.AlterField(
-            model_name="countrydri",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="countrydri",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="countryscholarship",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="countryscholarship",
-            name="other_advantages",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="countryscholarship",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="countrytaggeditem",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="countrytaggeditem",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="currency",
-            name="symbol",
-            field=models.CharField(blank=True, default="", max_length=30),
-        ),
-        migrations.AlterField(
-            model_name="previousdeparturefeedback",
-            name="adequation_comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="previousdeparturefeedback",
-            name="integration_comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="recommendation",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="university",
-            name="acronym",
-            field=models.CharField(blank=True, default="", max_length=20),
-        ),
-        migrations.AlterField(
-            model_name="university",
-            name="logo",
-            field=models.URLField(
-                blank=True,
-                default="",
-                validators=[
-                    backend_app.models.university.university.validate_extension_django
-                ],
-            ),
-        ),
-        migrations.AlterField(
-            model_name="university",
-            name="website",
-            field=models.URLField(blank=True, default="", max_length=300),
-        ),
-        migrations.AlterField(
-            model_name="universitydri",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="universitydri",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="universityinfo",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="universityinfo",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="universityscholarship",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="universityscholarship",
-            name="other_advantages",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="universityscholarship",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="universitysemestersdates",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="universitysemestersdates",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-        migrations.AlterField(
-            model_name="universitytaggeditem",
-            name="comment",
-            field=models.CharField(blank=True, default="", max_length=5000),
-        ),
-        migrations.AlterField(
-            model_name="universitytaggeditem",
-            name="title",
-            field=models.CharField(blank=True, default="", max_length=150),
-        ),
-    ]

backend/backend_app/migrations/0003_auto_20180916_1624.py

-# Generated by Django 2.0.3 on 2018-09-16 14:24
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [("backend_app", "0002_auto_20180916_1555")]
-
-    operations = [
-        migrations.AlterField(
-            model_name="countryscholarship",
-            name="frequency",
-            field=models.CharField(
-                blank=True,
-                choices=[
-                    ("w", "week"),
-                    ("m", "month"),
-                    ("s", "semester"),
-                    ("y", "year"),
-                    ("o", "one_shot"),
-                ],
-                default="m",
-                max_length=1,
-                null=True,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="universityscholarship",
-            name="frequency",
-            field=models.CharField(
-                blank=True,
-                choices=[
-                    ("w", "week"),
-                    ("m", "month"),
-                    ("s", "semester"),
-                    ("y", "year"),
-                    ("o", "one_shot"),
-                ],
-                default="m",
-                max_length=1,
-                null=True,
-            ),
-        ),
-    ]

backend/backend_app/migrations/0004_auto_20190310_1151.py

-# Generated by Django 2.1.7 on 2019-03-10 10:51
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [("backend_app", "0003_auto_20180916_1624")]
-
-    operations = [
-        migrations.RemoveField(model_name="semester", name="moderated_by"),
-        migrations.RemoveField(model_name="semester", name="updated_by"),
-        migrations.RemoveField(model_name="userdata", name="black_list"),
-        migrations.AddField(
-            model_name="offer",
-            name="year",
-            field=models.PositiveIntegerField(default=2018),
-        ),
-        migrations.AddField(
-            model_name="previousdeparture",
-            name="year",
-            field=models.PositiveIntegerField(default=2018),
-        ),
-        migrations.AlterField(
-            model_name="offer",
-            name="semester",
-            field=models.CharField(
-                choices=[("a", "autumn"), ("p", "spring")], default="a", max_length=2
-            ),
-        ),
-        migrations.AlterField(
-            model_name="previousdeparture",
-            name="semester",
-            field=models.CharField(
-                choices=[("a", "autumn"), ("p", "spring")], default="a", max_length=2
-            ),
-        ),
-        migrations.AlterUniqueTogether(name="offer", unique_together=set()),
-        migrations.DeleteModel(name="Semester"),
-    ]

backend/backend_app/models/abstract/my_model/__init__.py

@@ -5,6 +5,7 @@ from .pendingModeration import (
     PendingModeration,
     PendingModerationSerializer,
     PendingModerationViewSet,
+    PendingModerationObjViewset,
 )
 from .forTestingModeration import (
     ForTestingModeration,
@@ -31,6 +32,7 @@ __all__ = [
     "PendingModeration",
     "PendingModerationSerializer",
     "PendingModerationViewSet",
+    "PendingModerationObjViewset",
     "ForTestingModeration",
     "ForTestingModerationSerializer",
     "ForTestingModerationViewSet",

backend/backend_app/models/abstract/my_model/myModel.py

-from django.contrib.auth.models import User
+from base_app.models import User
 from django.contrib.contenttypes.fields import GenericRelation
 from django.core.exceptions import ValidationError
 from django.core.validators import MinValueValidator
@@ -48,6 +48,9 @@ class MyModel(models.Model):
     # Add the link to pending moderation
     pending_moderation = GenericRelation(PendingModeration)
 
+    # A bit of optimization: we store if there is something pending moderation
+    has_pending_moderation = models.BooleanField(default=False)
+
     class Meta:
         abstract = True
 

backend/backend_app/models/abstract/my_model/myModelSerializer.py

@@ -9,7 +9,7 @@ from rest_framework.validators import ValidationError
 from shared.obj_moderation_permission import DEFAULT_OBJ_MODERATION_LV
 
 from .myModel import MyModel
-from .pendingModeration import PendingModeration, PendingModerationSerializer
+from .pendingModeration import PendingModeration
 
 CLEANED_MY_MODEL_DATA = {
     "moderated_by": None,
@@ -44,7 +44,7 @@ class MyModelSerializer(MySerializerWithJSON):
     moderated_by = serializers.CharField(read_only=True)
     moderated_on = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)
 
-    pending_moderation = serializers.SerializerMethodField()
+    has_pending_moderation = serializers.BooleanField(read_only=True)
     model_config = serializers.SerializerMethodField()
 
     # For easier handling on the client side, we force an id field
@@ -59,27 +59,6 @@ class MyModelSerializer(MySerializerWithJSON):
 
         return self.Meta.model.model_config
 
-    # Optimization
-    # Class attribute to force the pending moderation to return the pending moderation data
-    # Or versionned element to return the number of versions
-    FORCE_FULL_DISPLAY = False
-
-    def get_pending_moderation(self, obj: MyModel):
-        """
-        Serializer for the `pending_moderation` field
-        """
-        # Optimization, in list mode, fetching all the pending moderation information is not optimal at all
-        if not self.FORCE_FULL_DISPLAY and self.context["view"].action == "list":
-            return None
-        else:
-            ct = ContentType.objects.get_for_model(self.Meta.model)
-            pending = PendingModeration.objects.filter(
-                content_type=ct, object_id=obj.pk
-            )
-            return PendingModerationSerializer(
-                pending, many=True, read_only=True, context=self.context
-            ).data
-
     def get_id(self, obj: MyModel):
         """
         Serializer for the id field.
@@ -193,6 +172,10 @@ class MyModelSerializer(MySerializerWithJSON):
                     "new_object": data_to_save,
                 },
             )
+
+            # Performance optimization, we store the fact that there is an object pending moderation
+            self.instance.has_pending_moderation = True
+            self.instance.save()
             return self.instance
 
         else:  # Moderation is not needed, we need to check whether it's a moderation or an update with no moderation
@@ -200,7 +183,7 @@ class MyModelSerializer(MySerializerWithJSON):
             moderated_and_updated = True
             if self.instance is None:
                 self.set_model_attr_no_moder(user, moderated_and_updated)
-                return super().save(*args, **kwargs)
+                instance = super().save(*args, **kwargs)
             else:
                 try:
                     pending_instance = PendingModeration.objects.get(
@@ -226,4 +209,9 @@ class MyModelSerializer(MySerializerWithJSON):
                     pass
 
                 self.set_model_attr_no_moder(user, moderated_and_updated)
-                return super().save(*args, **kwargs)
+                instance = super().save(*args, **kwargs)
+
+            # Performance optimization to know if has pending moderation
+            instance.has_pending_moderation = False
+            instance.save()
+            return instance

backend/backend_app/models/abstract/my_model/myModelVersionned.py

 from django.contrib.contenttypes.models import ContentType
 from django.core import serializers as djangoSerializers
 from django.core.serializers.base import DeserializationError
+from django.db import models
 
 import reversion
 from backend_app.custom import MySerializerWithJSON
@@ -20,6 +21,9 @@ class MyModelVersionned(MyModel):
     Custom MyModel that will be versionned in the app
     """
 
+    # We store the current number of versions for better performance
+    nb_versions = models.PositiveIntegerField(default=0)
+
     @classmethod
     def get_serializer(cls):
         """
@@ -39,20 +43,11 @@ class MyModelVersionnedSerializer(MyModelSerializer):
     """
 
     # Add a nb_versions field
-    nb_versions = serializers.SerializerMethodField()
+    nb_versions = serializers.IntegerField(read_only=True)
+
     # Add a content_type_id field to be able to find versions
     content_type_id = serializers.SerializerMethodField()
 
-    def get_nb_versions(self, obj):
-        """
-        Serializer for the nb_version field
-        With a bit of optimization
-        """
-        if self.FORCE_FULL_DISPLAY or self.context["view"].action != "list":
-            versions = Version.objects.get_for_object(obj)
-            return len(versions)
-        return None
-
     def get_content_type_id(self, obj):
         """
         Serializer for content type

backend/backend_app/models/abstract/my_model/myModelViewSet.py

@@ -40,6 +40,4 @@ class MyModelViewSet(viewsets.ModelViewSet):
         Extended default rest framework behavior
         to prefetch some table and enhance performances
         """
-        return self.queryset.prefetch_related(
-            "moderated_by", "updated_by", "pending_moderation"
-        )
+        return self.queryset.prefetch_related("moderated_by", "updated_by")

backend/backend_app/models/abstract/my_model/pendingModeration.py

-from django.contrib.auth.models import User
+from base_app.models import User
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
@@ -6,7 +6,7 @@ from django.db import models
 from backend_app.custom import MySerializerWithJSON
 from backend_app.fields import JSONField
 from backend_app.utils import get_model_config, get_viewset_permissions
-from rest_framework import serializers, viewsets
+from rest_framework import serializers, viewsets, mixins
 
 
 class PendingModeration(models.Model):
@@ -56,3 +56,19 @@ class PendingModerationViewSet(viewsets.ModelViewSet):
     permission_classes = get_viewset_permissions("PendingModerationViewSet")
     queryset = PendingModeration.objects.all()  # pylint: disable=E1101
     serializer_class = PendingModerationSerializer
+
+
+class PendingModerationObjViewset(mixins.ListModelMixin, viewsets.GenericViewSet):
+    """
+    Viewset to retrieve the pending moderation data for an object
+    """
+
+    permission_classes = get_viewset_permissions("PendingModerationObjViewset")
+    serializer_class = PendingModerationSerializer
+
+    def get_queryset(self):
+        content_type_id = self.kwargs["content_type_id"]
+        object_pk = self.kwargs["object_pk"]
+        return PendingModeration.objects.filter(
+            content_type=content_type_id, object_id=object_pk
+        )

backend/backend_app/models/abstract/scholarship/scholarship.py

@@ -53,8 +53,6 @@ class ScholarshipSerializer(BasicModuleSerializer):
     Serializer for the scholarship class
     """
 
-    FORCE_FULL_DISPLAY = True
-
     def validate(self, attrs):
         """
         Custom attribute validation

backend/backend_app/models/abstract/taggedItem/taggedItem.py

@@ -27,8 +27,6 @@ class TaggedItemSerializer(BasicModuleSerializer):
     Serializer for tagged items
     """
 
-    FORCE_FULL_DISPLAY = True
-
     def validate(self, attrs):
         attrs = super().validate(attrs)
 

backend/backend_app/models/country/countryDri.py

@@ -18,8 +18,6 @@ class CountryDri(BasicModule):
 
 
 class CountryDriSerializer(BasicModuleSerializer):
-    FORCE_FULL_DISPLAY = True
-
     class Meta:
         model = CountryDri
         fields = "__all__"

backend/backend_app/models/university/universityDri.py

@@ -18,8 +18,6 @@ class UniversityDri(BasicModule):
 
 
 class UniversityDriSerializer(BasicModuleSerializer):
-    FORCE_FULL_DISPLAY = True
-
     class Meta:
         model = UniversityDri
         fields = "__all__"

backend/backend_app/models/university/universityScholarship.py

@@ -20,8 +20,6 @@ class UniversityScholarship(Scholarship):
 
 
 class UniversityScholarshipSerializer(ScholarshipSerializer):
-    FORCE_FULL_DISPLAY = True
-
     class Meta:
         model = UniversityScholarship
         fields = "__all__"

backend/backend_app/models/user/previousDeparture.py

 from django.db import models
 from backend_app.models.other_core.specialty import Specialty
 from backend_app.models.university import University
-from django.contrib.auth.models import User
+from base_app.models import User
 from backend_app.fields import JSONField
 from backend_app.models.abstract.my_model import (
     MyModel,

backend/backend_app/models/user/userData.py

-from django.contrib.auth.models import User
+from base_app.models import User
 from django.db import models
 from backend_app.fields import JSONField
 from backend_app.models.abstract.my_model import (

backend/backend_app/models/user/userRestrictedModule.py

 from django.db import models
 from rest_framework import serializers
-from django.contrib.auth.models import User
+from base_app.models import User
 from backend_app.models.abstract.my_model import (
     MyModel,
     MyModelSerializer,

backend/backend_app/permissions/__list_user_post_permission.py

 from typing import List
 
-from django.contrib.auth.models import User
+from base_app.models import User
 
 from shared import get_api_objs
 

backend/backend_app/signals/__create_user_modules_post_create.py

 from django.db.models.signals import post_save
 
-from django.contrib.auth.models import User
+from base_app.models import User
 from backend_app.models.user import UserData
 
 

backend/backend_app/signals/__squash_revision_by_user.py

@@ -24,6 +24,11 @@ def squash_revision_by_user(sender, obj, **kwargs):
             else:
                 break
 
+    # We update the number of versions directly here
+    # So that it doesn't have to be recomputed every time
+    obj.nb_versions = len(Version.objects.get_for_object(obj))
+    obj.save()
+
 
 new_revision_saved.connect(
     squash_revision_by_user, dispatch_uid="receiver_concat_revisions"

backend/backend_app/tests/test_moderation.py

@@ -63,6 +63,9 @@ class ModerationTestCase(WithUserTestCase):
         self._test_val_null(instance.updated_on, null)
         self._test_val_null(instance.updated_by, null)
 
+    def _test_has_pending_moderation_val(self, instance, has: bool):
+        self.assertEqual(instance.has_pending_moderation, has)
+
     def _test_new_obj_val(self, instance, data):
         self.assertEqual(instance.new_object["aaa"], data["aaa"])
 
@@ -84,6 +87,7 @@ class ModerationTestCase(WithUserTestCase):
 
         self._test_updated_val(instance, null=True)
         self._test_moderated_val(instance, null=True)
+        self._test_has_pending_moderation_val(instance, True)
 
         instance_in_moderation = self._test_retreive_instance_in_moderation(instance)
         self._test_updated_val(instance_in_moderation, null=False)
@@ -97,6 +101,7 @@ class ModerationTestCase(WithUserTestCase):