Skip to content

GitLab

Commit 5530363c authored by Florent Chehab's avatar Florent Chehab
Browse files

Merge branch 'performance/re_setup_analysis' into 'master' 

Performance optimization

Closes #55 and #54

See merge request !65
parents b740f2dd 973b91f9
Pipeline #36225 passed with stages
in 6 minutes and 26 seconds
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1086 additions and 2323 deletions
backend/backend_app/load_data/loading_scripts/loadAdminUser.py
View file @ 5530363c
from django.contrib.auth.models import User
from base_app.models import User
import os
......
backend/backend_app/load_data/loading_scripts/loadCountries.py
View file @ 5530363c
import os
from django.contrib.auth.models import User
from base_app.models import User
import pandas as pd
from backend_app.models.country import Country
......
backend/backend_app/load_data/loading_scripts/loadCurrencies.py
View file @ 5530363c
......@@ -2,7 +2,7 @@ import csv
import os
from decimal import Decimal
from django.contrib.auth.models import User
from base_app.models import User
from backend_app.models.currency import Currency
......
backend/backend_app/load_data/loading_scripts/loadGeneric.py
View file @ 5530363c
from django.contrib.auth.models import User
from base_app.models import User
from django.utils import timezone
import reversion
......
backend/backend_app/load_data/loading_scripts/loadTags.py
View file @ 5530363c
import json
import os
from django.contrib.auth.models import User
from base_app.models import User
from backend_app.models.tag import Tag
......
backend/backend_app/load_data/loading_scripts/loadUniversities.py
View file @ 5530363c
import os
from django.contrib.auth.models import User
from base_app.models import User
import pandas as pd
from backend_app.models.campus import Campus
......
backend/backend_app/load_data/loading_scripts/loadUniversityEx.py
View file @ 5530363c
from datetime import datetime
from django.contrib.auth.models import User
from base_app.models import User
from backend_app.models.country import Country, CountryScholarship
from backend_app.models.currency import Currency
......
backend/backend_app/migrations/0001_initial.py deleted 100644 → 0
View file @ b740f2dd
This diff is collapsed.
backend/backend_app/migrations/0001_new-initial.py 0 → 100644
View file @ 5530363c
This diff is collapsed.
backend/backend_app/migrations/0002_auto_20180916_1555.py deleted 100644 → 0
View file @ b740f2dd
# Generated by Django 2.0.3 on 2018-09-16 13:55
import backend_app.models.university.university
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("backend_app", "0001_initial")]
operations = [
migrations.AlterField(
model_name="campus",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="campus",
name="name",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="campus",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="campustaggeditem",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="campustaggeditem",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="city",
name="area",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="city",
name="local_name",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="citytaggeditem",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="citytaggeditem",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="country",
name="intermediate_region_name",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="country",
name="intermediate_region_un_code",
field=models.CharField(blank=True, default="", max_length=3),
),
migrations.AlterField(
model_name="country",
name="iso_alpha3_code",
field=models.CharField(blank=True, default="", max_length=3, unique=True),
),
migrations.AlterField(
model_name="country",
name="sub_region_name",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="country",
name="sub_region_un_code",
field=models.CharField(blank=True, default="", max_length=3),
),
migrations.AlterField(
model_name="countrydri",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="countrydri",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="countryscholarship",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="countryscholarship",
name="other_advantages",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="countryscholarship",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="countrytaggeditem",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="countrytaggeditem",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="currency",
name="symbol",
field=models.CharField(blank=True, default="", max_length=30),
),
migrations.AlterField(
model_name="previousdeparturefeedback",
name="adequation_comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="previousdeparturefeedback",
name="integration_comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="recommendation",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="university",
name="acronym",
field=models.CharField(blank=True, default="", max_length=20),
),
migrations.AlterField(
model_name="university",
name="logo",
field=models.URLField(
blank=True,
default="",
validators=[
backend_app.models.university.university.validate_extension_django
],
),
),
migrations.AlterField(
model_name="university",
name="website",
field=models.URLField(blank=True, default="", max_length=300),
),
migrations.AlterField(
model_name="universitydri",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="universitydri",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="universityinfo",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="universityinfo",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="universityscholarship",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="universityscholarship",
name="other_advantages",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="universityscholarship",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="universitysemestersdates",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="universitysemestersdates",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
migrations.AlterField(
model_name="universitytaggeditem",
name="comment",
field=models.CharField(blank=True, default="", max_length=5000),
),
migrations.AlterField(
model_name="universitytaggeditem",
name="title",
field=models.CharField(blank=True, default="", max_length=150),
),
]
backend/backend_app/migrations/0003_auto_20180916_1624.py deleted 100644 → 0
View file @ b740f2dd
# Generated by Django 2.0.3 on 2018-09-16 14:24
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("backend_app", "0002_auto_20180916_1555")]
operations = [
migrations.AlterField(
model_name="countryscholarship",
name="frequency",
field=models.CharField(
blank=True,
choices=[
("w", "week"),
("m", "month"),
("s", "semester"),
("y", "year"),
("o", "one_shot"),
],
default="m",
max_length=1,
null=True,
),
),
migrations.AlterField(
model_name="universityscholarship",
name="frequency",
field=models.CharField(
blank=True,
choices=[
("w", "week"),
("m", "month"),
("s", "semester"),
("y", "year"),
("o", "one_shot"),
],
default="m",
max_length=1,
null=True,
),
),
]
backend/backend_app/migrations/0004_auto_20190310_1151.py deleted 100644 → 0
View file @ b740f2dd
# Generated by Django 2.1.7 on 2019-03-10 10:51
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("backend_app", "0003_auto_20180916_1624")]
operations = [
migrations.RemoveField(model_name="semester", name="moderated_by"),
migrations.RemoveField(model_name="semester", name="updated_by"),
migrations.RemoveField(model_name="userdata", name="black_list"),
migrations.AddField(
model_name="offer",
name="year",
field=models.PositiveIntegerField(default=2018),
),
migrations.AddField(
model_name="previousdeparture",
name="year",
field=models.PositiveIntegerField(default=2018),
),
migrations.AlterField(
model_name="offer",
name="semester",
field=models.CharField(
choices=[("a", "autumn"), ("p", "spring")], default="a", max_length=2
),
),
migrations.AlterField(
model_name="previousdeparture",
name="semester",
field=models.CharField(
choices=[("a", "autumn"), ("p", "spring")], default="a", max_length=2
),
),
migrations.AlterUniqueTogether(name="offer", unique_together=set()),
migrations.DeleteModel(name="Semester"),
]
backend/backend_app/models/abstract/my_model/__init__.py
View file @ 5530363c
......@@ -5,6 +5,7 @@ from .pendingModeration import (
PendingModeration,
PendingModerationSerializer,
PendingModerationViewSet,
PendingModerationObjViewset,
)
from .forTestingModeration import (
ForTestingModeration,
......@@ -31,6 +32,7 @@ __all__ = [
"PendingModeration",
"PendingModerationSerializer",
"PendingModerationViewSet",
"PendingModerationObjViewset",
"ForTestingModeration",
"ForTestingModerationSerializer",
"ForTestingModerationViewSet",
......
backend/backend_app/models/abstract/my_model/myModel.py
View file @ 5530363c
from django.contrib.auth.models import User
from base_app.models import User
from django.contrib.contenttypes.fields import GenericRelation
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator
......@@ -48,6 +48,9 @@ class MyModel(models.Model):
# Add the link to pending moderation
pending_moderation = GenericRelation(PendingModeration)
# A bit of optimization: we store if there is something pending moderation
has_pending_moderation = models.BooleanField(default=False)
class Meta:
abstract = True
......
backend/backend_app/models/abstract/my_model/myModelSerializer.py
View file @ 5530363c
......@@ -9,7 +9,7 @@ from rest_framework.validators import ValidationError
from shared.obj_moderation_permission import DEFAULT_OBJ_MODERATION_LV
from .myModel import MyModel
from .pendingModeration import PendingModeration, PendingModerationSerializer
from .pendingModeration import PendingModeration
CLEANED_MY_MODEL_DATA = {
"moderated_by": None,
......@@ -44,7 +44,7 @@ class MyModelSerializer(MySerializerWithJSON):
moderated_by = serializers.CharField(read_only=True)
moderated_on = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)
pending_moderation = serializers.SerializerMethodField()
has_pending_moderation = serializers.BooleanField(read_only=True)
model_config = serializers.SerializerMethodField()
# For easier handling on the client side, we force an id field
......@@ -59,27 +59,6 @@ class MyModelSerializer(MySerializerWithJSON):
return self.Meta.model.model_config
# Optimization
# Class attribute to force the pending moderation to return the pending moderation data
# Or versionned element to return the number of versions
FORCE_FULL_DISPLAY = False
def get_pending_moderation(self, obj: MyModel):
"""
Serializer for the `pending_moderation` field
"""
# Optimization, in list mode, fetching all the pending moderation information is not optimal at all
if not self.FORCE_FULL_DISPLAY and self.context["view"].action == "list":
return None
else:
ct = ContentType.objects.get_for_model(self.Meta.model)
pending = PendingModeration.objects.filter(
content_type=ct, object_id=obj.pk
)
return PendingModerationSerializer(
pending, many=True, read_only=True, context=self.context
).data
def get_id(self, obj: MyModel):
"""
Serializer for the id field.
......@@ -193,6 +172,10 @@ class MyModelSerializer(MySerializerWithJSON):
"new_object": data_to_save,
},
)
# Performance optimization, we store the fact that there is an object pending moderation
self.instance.has_pending_moderation = True
self.instance.save()
return self.instance
else: # Moderation is not needed, we need to check whether it's a moderation or an update with no moderation
......@@ -200,7 +183,7 @@ class MyModelSerializer(MySerializerWithJSON):
moderated_and_updated = True
if self.instance is None:
self.set_model_attr_no_moder(user, moderated_and_updated)
return super().save(*args, **kwargs)
instance = super().save(*args, **kwargs)
else:
try:
pending_instance = PendingModeration.objects.get(
......@@ -226,4 +209,9 @@ class MyModelSerializer(MySerializerWithJSON):
pass
self.set_model_attr_no_moder(user, moderated_and_updated)
return super().save(*args, **kwargs)
instance = super().save(*args, **kwargs)
# Performance optimization to know if has pending moderation
instance.has_pending_moderation = False
instance.save()
return instance
backend/backend_app/models/abstract/my_model/myModelVersionned.py
View file @ 5530363c
from django.contrib.contenttypes.models import ContentType
from django.core import serializers as djangoSerializers
from django.core.serializers.base import DeserializationError
from django.db import models
import reversion
from backend_app.custom import MySerializerWithJSON
......@@ -20,6 +21,9 @@ class MyModelVersionned(MyModel):
Custom MyModel that will be versionned in the app
"""
# We store the current number of versions for better performance
nb_versions = models.PositiveIntegerField(default=0)
@classmethod
def get_serializer(cls):
"""
......@@ -39,20 +43,11 @@ class MyModelVersionnedSerializer(MyModelSerializer):
"""
# Add a nb_versions field
nb_versions = serializers.SerializerMethodField()
nb_versions = serializers.IntegerField(read_only=True)
# Add a content_type_id field to be able to find versions
content_type_id = serializers.SerializerMethodField()
def get_nb_versions(self, obj):
"""
Serializer for the nb_version field
With a bit of optimization
"""
if self.FORCE_FULL_DISPLAY or self.context["view"].action != "list":
versions = Version.objects.get_for_object(obj)
return len(versions)
return None
def get_content_type_id(self, obj):
"""
Serializer for content type
......
backend/backend_app/models/abstract/my_model/myModelViewSet.py
View file @ 5530363c
......@@ -40,6 +40,4 @@ class MyModelViewSet(viewsets.ModelViewSet):
Extended default rest framework behavior
to prefetch some table and enhance performances
"""
return self.queryset.prefetch_related(
"moderated_by", "updated_by", "pending_moderation"
)
return self.queryset.prefetch_related("moderated_by", "updated_by")
backend/backend_app/models/abstract/my_model/pendingModeration.py
View file @ 5530363c
from django.contrib.auth.models import User
from base_app.models import User
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
......@@ -6,7 +6,7 @@ from django.db import models
from backend_app.custom import MySerializerWithJSON
from backend_app.fields import JSONField
from backend_app.utils import get_model_config, get_viewset_permissions
from rest_framework import serializers, viewsets
from rest_framework import serializers, viewsets, mixins
class PendingModeration(models.Model):
......@@ -56,3 +56,19 @@ class PendingModerationViewSet(viewsets.ModelViewSet):
permission_classes = get_viewset_permissions("PendingModerationViewSet")
queryset = PendingModeration.objects.all() # pylint: disable=E1101
serializer_class = PendingModerationSerializer
class PendingModerationObjViewset(mixins.ListModelMixin, viewsets.GenericViewSet):
"""
Viewset to retrieve the pending moderation data for an object
"""
permission_classes = get_viewset_permissions("PendingModerationObjViewset")
serializer_class = PendingModerationSerializer
def get_queryset(self):
content_type_id = self.kwargs["content_type_id"]
object_pk = self.kwargs["object_pk"]
return PendingModeration.objects.filter(
content_type=content_type_id, object_id=object_pk
)
backend/backend_app/models/abstract/scholarship/scholarship.py
View file @ 5530363c
......@@ -53,8 +53,6 @@ class ScholarshipSerializer(BasicModuleSerializer):
Serializer for the scholarship class
"""
FORCE_FULL_DISPLAY = True
def validate(self, attrs):
"""
Custom attribute validation
......
backend/backend_app/models/abstract/taggedItem/taggedItem.py
View file @ 5530363c
......@@ -27,8 +27,6 @@ class TaggedItemSerializer(BasicModuleSerializer):
Serializer for tagged items
"""
FORCE_FULL_DISPLAY = True