Commit 4fe1bdf2 authored by Florent Chehab's avatar Florent Chehab

Cleaning

parent 0e42118e
......@@ -15,4 +15,7 @@ indent_size = 4
indent_size = 2
[*.json]
indent_size = 2
\ No newline at end of file
indent_size = 2
[Makefile]
indent_style = tab
......@@ -9,6 +9,9 @@ docker-pull:
up--build:
docker-compose up --build
reformat_backend:
docker-compose exec backend sh -c "cd backend && black ."
test_backend:
docker-compose exec backend sh -c "cd backend && pytest base_app/ frontend_app/ backend_app/"
......
......@@ -7,31 +7,44 @@ import importlib
api_config = get_api_config()
CLASSIC_MODELS = []
# models that are versionned in the app
VERSIONNED_MODELS = []
# Other models, ie not versionned
CLASSIC_MODELS = []
for model in api_config:
if "model" in model and model["model"]:
model = DotMap(model)
if (not model.requires_testing) and (not model.ignore_in_admin):
# Go through the API configuraion
for entry in api_config:
if "model" in entry and entry["model"]:
model_obj = DotMap(entry)
if (not model_obj.requires_testing) and (not model_obj.ignore_in_admin):
# Import the model
module = importlib.import_module(
"backend_app.models.{}".format(model.import_location)
"backend_app.models.{}".format(model_obj.import_location)
)
if model.versionned:
VERSIONNED_MODELS.append(getattr(module, model.model))
# Add it to the correct list of models
if model_obj.versionned:
VERSIONNED_MODELS.append(getattr(module, model_obj.model))
else:
CLASSIC_MODELS.append(getattr(module, model.model))
CLASSIC_MODELS.append(getattr(module, model_obj.model))
#######
# Some dynamic checks
#######
for model in CLASSIC_MODELS:
admin.site.register(model)
for Model in CLASSIC_MODELS:
# Register the model in the admin in a standard way
admin.site.register(Model)
try:
model.get_serializer()
raise Exception("CLASSIC MODEL SHOULDN'T have this method")
# Check that it doesn't have the get_serializer method
Model.get_serializer()
raise Exception("A 'CLASSIC MODEL' SHOULDN'T have the get_serializer method")
except AttributeError:
pass
for model in VERSIONNED_MODELS:
admin.site.register(model, CompareVersionAdmin)
if model.get_serializer().Meta.model != model:
raise Exception("Get_serializer configuration incorrect in", str(model))
for Model in VERSIONNED_MODELS:
# Register the model in the admin with versioning
admin.site.register(Model, CompareVersionAdmin)
# Check that it has a get_serializer method
if Model.get_serializer().Meta.model != Model:
raise Exception("Get_serializer configuration incorrect in", str(Model))
......@@ -2,12 +2,13 @@ from backend_app.fields import JSONField
from rest_framework import serializers
field_mapping = serializers.ModelSerializer.serializer_field_mapping
# Small hack to register our custom JSONField class as a regular JSONfield
field_mapping[JSONField] = serializers.JSONField
class MySerializerWithJSON(serializers.ModelSerializer):
"""
Simple class to add support for custom JSONField support
Simple class to add serializing support for custom JSONField
"""
serializer_field_mapping = field_mapping
#!/usr/bin/env python3
"""
Script to insert the country data in the database
IT HAS TO BE RUN INSIDE ./manage.py shell
TODO YURK. Use pandas @florent !!
"""
import csv
import os
import time
from geopy.geocoders import Nominatim
import reverse_geocoder as rg
from geopy.geocoders import Nominatim
tmp = os.path.join(os.path.realpath(__file__), "../../assets/destinations.csv")
destinations_path = os.path.abspath(tmp)
......
from .loading_scripts import LoadGroups
from .loading_scripts import LoadAdminUser
from .loading_scripts import LoadCurrencies
from .loading_scripts import LoadCountries
from .loading_scripts import LoadUniversities
from .loading_scripts import LoadTags
from .loading_scripts import LoadUniversityEx
import reversion
from .loading_scripts import (
LoadAdminUser,
LoadCountries,
LoadCurrencies,
LoadGroups,
LoadTags,
LoadUniversities,
LoadUniversityEx,
)
def load_all():
"""Function to load all the initial data in the app
"""
with reversion.create_revision():
LoadGroups()
admin = LoadAdminUser().get()
......
......@@ -15,3 +15,5 @@ __all__ = [
"LoadCurrencies",
"LoadUniversityEx",
]
# TODO in all loading files, use a path to the assets folder define somewhere else.
......@@ -22,5 +22,5 @@ class LoadAdminUser(object):
username=os.environ["DJANGO_ADMIN_USERNAME"]
)[0]
def get(self):
def get(self) -> User:
return self.admin
from backend_app.models.country import Country
import os
from django.contrib.auth.models import User
import pandas as pd
from backend_app.models.country import Country
from .loadGeneric import LoadGeneric
class LoadCountries(LoadGeneric):
def __init__(self, admin):
"""
Class to handle the loading of countries in the app
"""
def __init__(self, admin: User):
self.admin = admin
def load(self):
......
from backend_app.models.currency import Currency
import os
import csv
from .loadGeneric import LoadGeneric
import os
from decimal import Decimal
from django.contrib.auth.models import User
from backend_app.models.currency import Currency
from .loadGeneric import LoadGeneric
class LoadCurrencies(LoadGeneric):
def __init__(self, admin):
"""
Load currencies in the app
"""
def __init__(self, admin: User):
self.admin = admin
def load(self):
......
from django.contrib.auth.models import User
from django.utils import timezone
import reversion
from backend_app.models.abstract.my_model import MyModel
class LoadGeneric(object):
"""Class to handle the loading of initial data in a generic fashion
"""
@classmethod
def add_info_and_save(cls, obj, admin):
def add_info_and_save(cls, obj: MyModel, admin: User):
with reversion.create_revision():
obj.moderated_by = admin
obj.updated_by = admin
......
......@@ -2,6 +2,9 @@ from django.contrib.auth.models import Group
class LoadGroups(object):
"""Class to add the default user groups to the app
"""
def __init__(self):
Group.objects.get_or_create(name="Moderators")
Group.objects.get_or_create(name="DRI")
import json
import os
from django.contrib.auth.models import User
from backend_app.models.tag import Tag
import os
import json
from .loadGeneric import LoadGeneric
class LoadTags(LoadGeneric):
def __init__(self, admin):
"""
Class to load the tags in the app.
"""
def __init__(self, admin: User):
self.admin = admin
def load(self):
......
from backend_app.models.country import Country
import os
from django.contrib.auth.models import User
import pandas as pd
from backend_app.models.campus import Campus
from backend_app.models.city import City
from backend_app.models.country import Country
from backend_app.models.university import University
from backend_app.models.campus import Campus
import os
import pandas as pd
from .loadGeneric import LoadGeneric
class LoadUniversities(LoadGeneric):
def __init__(self, admin):
"""
Load the universities in the app
"""
def __init__(self, admin: User):
self.admin = admin
def load(self):
......
from .loadGeneric import LoadGeneric
from backend_app.models.university import University
from backend_app.models.university import UniversityDri
from backend_app.models.university import UniversityInfo
from backend_app.models.university import UniversitySemestersDates
from backend_app.models.country import CountryScholarship
from backend_app.models.country import Country
from backend_app.models.university import UniversityTaggedItem
from datetime import datetime
from django.contrib.auth.models import User
from backend_app.models.country import Country, CountryScholarship
from backend_app.models.currency import Currency
from backend_app.models.tag import Tag
from backend_app.models.university import (
University,
UniversityDri,
UniversityInfo,
UniversitySemestersDates,
UniversityTaggedItem,
)
from datetime import datetime
from .loadGeneric import LoadGeneric
class LoadUniversityEx(LoadGeneric):
def __init__(self, admin):
"""
Load some exemple data for the EPFL
"""
def __init__(self, admin: User):
self.admin = admin
def load(self):
......
from django.db import models
from backend_app.fields import JSONField
from backend_app.models.abstract.my_model import (
MyModelVersionned,
MyModelVersionnedSerializer,
MyModelVersionnedViewSet,
)
from backend_app.fields import JSONField
from backend_app.validators.tag import validate_content_against_config
from backend_app.validators.tag.tags_config import USEFULL_LINKS_CONFIG
......@@ -12,6 +13,15 @@ IMPORTANCE_LEVEL = (("-", "normal"), ("+", "important"), ("++", "IMPORTANT"))
class BasicModule(MyModelVersionned):
"""
Abstract module that provides defaults fields:
Title, comment, useful_links and importance_level
Those field will be inherited.
All Basic modules are also "versionned" modules
"""
title = models.CharField(default="", blank=True, max_length=150)
comment = models.CharField(default="", blank=True, max_length=5000)
useful_links = JSONField(default=list)
......@@ -24,7 +34,15 @@ class BasicModule(MyModelVersionned):
class BasicModuleSerializer(MyModelVersionnedSerializer):
"""
Custom serializer that performs checks on the Basic module filed
"""
def my_validate(self, attrs):
"""
Checks that the useful_links have been filled properly
"""
content = {"useful_links": attrs["useful_links"]}
config = {"useful_links": USEFULL_LINKS_CONFIG}
validate_content_against_config(config, content)
......@@ -36,4 +54,8 @@ class BasicModuleSerializer(MyModelVersionnedSerializer):
class BasicModuleViewSet(MyModelVersionnedViewSet):
"""
Viewset for the Basic Module
"""
serializer_class = BasicModuleSerializer
......@@ -12,6 +12,7 @@ from .forTestingModeration import (
ForTestingModerationViewSet,
)
from .myModelVersionned import (
Version,
MyModelVersionned,
MyModelVersionnedSerializer,
MyModelVersionnedViewSet,
......@@ -33,6 +34,7 @@ __all__ = [
"ForTestingModeration",
"ForTestingModerationSerializer",
"ForTestingModerationViewSet",
"Version",
"MyModelVersionned",
"MyModelVersionnedSerializer",
"MyModelVersionnedViewSet",
......
from django.db import models
from backend_app.utils import get_model_config, get_viewset_permissions
from .myModel import MyModel
from .myModelSerializer import MyModelSerializer
from .myModelViewSet import MyModelViewSet
from django.db import models
from backend_app.utils import get_model_config, get_viewset_permissions
class ForTestingModeration(MyModel):
......@@ -16,7 +18,7 @@ class ForTestingModeration(MyModel):
class ForTestingModerationSerializer(MyModelSerializer):
"""
Same as above
Simple serializer for testing purpose
"""
class Meta:
......@@ -26,7 +28,7 @@ class ForTestingModerationSerializer(MyModelSerializer):
class ForTestingModerationViewSet(MyModelViewSet):
"""
Same as above
Simple Viewset for testing purpose
"""
permission_classes = get_viewset_permissions("ForTestingModerationViewSet")
......
from django.db import models
import reversion
from backend_app.utils import get_model_config, get_viewset_permissions
from .myModelVersionned import (
MyModelVersionned,
MyModelVersionnedSerializer,
MyModelVersionnedViewSet,
)
from django.db import models
import reversion
from backend_app.utils import get_model_config, get_viewset_permissions
@reversion.register()
class ForTestingVersioning(MyModelVersionned):
"""
Simple model for testing purposes
Simple model for testing purposes (versioning)
"""
model_config = get_model_config("ForTestingVersioning")
......@@ -24,7 +26,7 @@ class ForTestingVersioning(MyModelVersionned):
class ForTestingVersioningSerializer(MyModelVersionnedSerializer):
"""
Same as above
Simple Serializer for testing purposes (versioning)
"""
class Meta:
......@@ -34,7 +36,7 @@ class ForTestingVersioningSerializer(MyModelVersionnedSerializer):
class ForTestingVersioningViewSet(MyModelVersionnedViewSet):
"""
Same as above
Simple Viewset for testing purposes (versioning)
"""
permission_classes = get_viewset_permissions("ForTestingVersioningViewSet")
......
from django.db import models
from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import GenericRelation
from .pendingModeration import PendingModeration
from shared import OBJ_MODERATION_PERMISSIONS
from shared import DEFAULT_OBJ_MODERATION_LV
from django.core.validators import MinValueValidator
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator
from django.db import models
from shared import DEFAULT_OBJ_MODERATION_LV, OBJ_MODERATION_PERMISSIONS
from .pendingModeration import PendingModeration
oml = DEFAULT_OBJ_MODERATION_LV
POSSIBLE_OBJ_MODER_LV = [
OBJ_MODERATION_PERMISSIONS[key] for key in OBJ_MODERATION_PERMISSIONS
]
......@@ -21,23 +21,31 @@ def validate_obj_model_lv(value):
class MyModel(models.Model):
"""
All models in the app deppend of this one.
It contains the required attributes for managing eventual moderation data.
It contains the required attributes for managing optionnal data moderation.
All the logic behind moderation is done in myModelSerializer
"""
moderated_by = models.ForeignKey(
# store the update author
updated_by = models.ForeignKey(
User, null=True, on_delete=models.SET_NULL, related_name="+"
)
moderated_on = models.DateTimeField(null=True)
# store the update date (model can be updated without moderation)
updated_on = models.DateTimeField(null=True)
updated_by = models.ForeignKey(
# store the moderator
moderated_by = models.ForeignKey(
User, null=True, on_delete=models.SET_NULL, related_name="+"
)
# store the moderation date
moderated_on = models.DateTimeField(null=True)
# Store the object moderation level by default
obj_moderation_level = models.SmallIntegerField(
default=oml, validators=[MinValueValidator(0), validate_obj_model_lv]
default=DEFAULT_OBJ_MODERATION_LV,
validators=[MinValueValidator(0), validate_obj_model_lv],
)
# Add the link to pending moderation
pending_moderation = GenericRelation(PendingModeration)
class Meta:
......
from django.contrib.contenttypes.models import ContentType
from django.utils import timezone
from backend_app.custom import MySerializerWithJSON
from backend_app.permissions import is_moderation_required
from backend_app.utils import get_user_level
from rest_framework import serializers
from rest_framework.validators import ValidationError
from django.utils import timezone
from .pendingModeration import PendingModeration
from django.contrib.contenttypes.models import ContentType
from .myModel import MyModel
from .pendingModeration import PendingModerationSerializer
from backend_app.utils import get_user_level
from backend_app.permissions import is_moderation_required
from backend_app.custom import MySerializerWithJSON
from .pendingModeration import PendingModeration, PendingModerationSerializer
CLEANED_MY_MODEL_DATA = {
"moderated_by": None,
......@@ -17,7 +18,10 @@ CLEANED_MY_MODEL_DATA = {
}
def override_data(old_data, new_data):
def override_data(old_data: dict, new_data: dict) -> dict:
"""Update the data in old_data with the one in new_data
"""
for key in new_data:
if key in old_data:
old_data[key] = new_data[key]
......@@ -25,10 +29,20 @@ def override_data(old_data, new_data):
class MyModelSerializer(MySerializerWithJSON):
moderated_on = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)
"""Serializer to go along the MyModel Model. This serializer handles backend data moderation checks and tricks.
Raises:
ValidationError -- If you are trying to moderate something you don't have rights to
"""
######
# Basic fields serializers
updated_by = serializers.CharField(read_only=True)
updated_on = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)
moderated_by = serializers.CharField(read_only=True)
updated_by = serializers.CharField(read_only=True)
moderated_on = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)
pending_moderation = serializers.SerializerMethodField()
model_config = serializers.SerializerMethodField()
......@@ -36,12 +50,24 @@ class MyModelSerializer(MySerializerWithJSON):
# this is useful when a model has a dedicated primary key
id = serializers.SerializerMethodField()
def get_model_config(self, obj=None):
def get_model_config(self, obj=None) -> dict:
"""
Serializer for the `model_config` field.
`obj` is required in the function definition, but it's not used.
"""
return self.Meta.model.model_config
# Optimization
# Class attribute to force the pending moderation to return the pending moderation data
# Or versionned element to return the number of versions
FORCE_FULL_DISPLAY = False
def get_pending_moderation(self, obj):
def get_pending_moderation(self, obj: MyModel):
"""
Serializer for the `pending_moderation` field
"""
# Optimization, in list mode, fetching all the pending moderation information is not optimal at all
if not self.FORCE_FULL_DISPLAY and self.context["view"].action == "list":
return None
else:
......@@ -53,19 +79,28 @@ class MyModelSerializer(MySerializerWithJSON):
pending, many=True, read_only=True, context=self.context
).data
def get_id(self, obj):
def get_id(self, obj: MyModel):
"""
Serializer for the id field.
"""
return obj.pk
class Meta:
model = MyModel
def my_validate(self, attrs):
"""
Function to be redefined in the subclasses to validate class fields.
"""
return attrs
def validate(self, attrs):
"""
Validate `MyModel` fields and enforce certain field at the backend level.
TODO unit test this
"""
# Enforce fields values based on request
self.user = self.context["request"].user
self.user_level = get_user_level(self.user)
......@@ -80,6 +115,10 @@ class MyModelSerializer(MySerializerWithJSON):
return self.my_validate(attrs)
def set_model_attr_no_moder(self, moderated_and_updated):
"""
TODO
TODO: rename ?
"""
now = timezone.now()
self.override_validated_data({"moderated_by": self.user, "moderated_on": now})
......@@ -87,9 +126,12 @@ class MyModelSerializer(MySerializerWithJSON):
self.override_validated_data({"updated_by": self.user, "updated_on": now})
def clean_validated_data(self):
"""
Clear fields related to update and moderation
"""
self.override_validated_data(CLEANED_MY_MODEL_DATA)
def override_validated_data(self, new_data):
def override_validated_data(self, new_data: dict):
"""
Method used to force specific attributes when saving a model
"""
......@@ -97,12 +139,23 @@ class MyModelSerializer(MySerializerWithJSON):
self.validated_data[key] = new_data[key]
def my_pre_save(self):
"""
TODO, Analyse if usefull
"""