Commit 26e608b7 authored by Florent Chehab's avatar Florent Chehab

feat(backend): huge redesign and simplification

* Removed `model_config` from all models; updated `MyModelSerializer` to include a new `obj_info` field. Updated frontend to take the change into account. (Fixes #78)
* Removed `get_viewset_permissions` from most viewsets and added a generic getter in `MyModelViewset`.
* Added support for composable permissions classes :confetti\_ball: (Fixes #45)
* Cleaned config files: separated the files; added `defaults.yaml`
* Moved `shared` folder (content) to `backend.backend_app.config` (still accessible to frontend, but it's cleaner that way since this folder contains files concerning only the backend).
* Performance update with caching some attributes;
* Even cleaner backend dynamic imports (#46)
* Added a good chunck of documentation related to the backend (#74)
* Added checks (runned when server is started or indirectly with `make check_backend`)
parent 298d4767
Pipeline #36659 passed with stages
in 4 minutes and 53 seconds
from django.contrib import admin
from reversion_compare.admin import CompareVersionAdmin
from shared import get_api_objs
VERSIONNED_MODELS = map(
lambda obj: obj.Model,
get_api_objs(
has_model=True, ignore_in_admin=False, versionned=True, requires_testing=False
),
)
CLASSIC_MODELS = map(
lambda obj: obj.Model,
get_api_objs(
has_model=True, ignore_in_admin=False, versionned=False, requires_testing=False
),
)
from backend_app.config.models import get_models
from backend_app.checks import check_classic_models, check_versionned_models
VERSIONNED_MODELS = get_models(versionned=True, requires_testing=False)
CLASSIC_MODELS = get_models(versionned=False, requires_testing=False)
#######
# Register the models and perform some dynamic checks
# Perform some dynamic checks
#######
check_classic_models(CLASSIC_MODELS)
check_versionned_models(VERSIONNED_MODELS)
#######
# Register the models
#######
for Model in CLASSIC_MODELS:
# Register the model in the admin in a standard way
admin.site.register(Model)
try:
# Check that it doesn't have the get_serializer method
Model.get_serializer()
raise Exception(
"A 'CLASSIC MODEL' SHOULDN'T have the "
"get_serializer method, {}".format(Model)
)
except AttributeError:
pass
for Model in VERSIONNED_MODELS:
# Register the model in the admin with versioning
admin.site.register(Model, CompareVersionAdmin)
# Check that it has a get_serializer method
if Model.get_serializer().Meta.model != Model:
raise Exception("Get_serializer configuration incorrect in", str(Model))
def check_classic_models(classic_models):
"""
Check that all "classic" models don't have a `get_serializer` method:
they don't need it.
See doc for more information:
http://localhost:5000/#/Application/Backend/models_serializers_viewsets
"""
for Model in classic_models:
try:
# Check that it doesn't have the get_serializer method
Model.get_serializer()
raise Exception(
"A 'CLASSIC MODEL' SHOULDN'T have the "
"get_serializer method, {}".format(Model)
)
except AttributeError:
pass
def check_versionned_models(versionned_models):
"""
Check that all "versionned" models have a `get_serializer` method.
See doc for more information:
http://localhost:5000/#/Application/Backend/models_serializers_viewsets
"""
for Model in versionned_models:
# Check that it has a get_serializer method
if Model.get_serializer().Meta.model != Model:
raise Exception("Get_serializer configuration incorrect in", str(Model))
DEFAULT_VIEWSET_PERMISSIONS: IsAuthenticated & (IsStaff | NoDelete)
DEFAULT_MODEL_MODERATION_LEVEL: 2
DEFAULT_OBJ_MODERATION_PERMISSIONS:
staff:
level: 3
label: Administrateurs et administratrices du site
DRI:
level: 2
label: Membres de la DRI
moderator:
level: 1
label: Modérateurs et modératrices
authenticated_user:
level: 0
label: Utilisateurs et utilisatrices authentifiées
import importlib
from typing import List, Optional, Union
from django.conf import settings
from backend_app.models.abstract.my_model.myModelVersionned import MyModelVersionned
from .utils import load_viewsets_config
def get_models(
versionned: Optional[bool] = None,
requires_testing: Union[None, bool, "smart"] = None,
) -> List[object]:
"""
Returns the list of models that could be ingered from the `viewsets_config.yml`
config file.
Some filtering can be applied to some attributes.
If the parameter is `None` then no filtering is applied to this attribute.
If the parameter is `True` or `False` the object is returned only if it matched.
There is one exception for the parameter `requires_testing` if it is set to `smart` then
the object is returned only if doesn't require testing or if testing is activated.
"""
out = list()
for obj in load_viewsets_config():
if requires_testing is not None:
if requires_testing == "smart":
if obj.requires_testing and not settings.TESTING:
continue
else:
if requires_testing and not settings.TESTING:
continue
if requires_testing and not obj.requires_testing:
continue
if not requires_testing and obj.requires_testing:
continue
module = importlib.import_module(
"backend_app.models.{}".format(obj.import_location)
)
Viewset = getattr(module, obj.viewset)
if not hasattr(Viewset, "serializer_class"):
# It is an API viewset, we don't care about it
# API viewsets don't correspond to a model.
continue
Model = Viewset.serializer_class.Meta.model
if versionned is not None:
if versionned and not issubclass(Model, MyModelVersionned):
continue
if not versionned and issubclass(Model, MyModelVersionned):
continue
out.append(Model)
return list(set(out)) # make sure we return each models at most once
# THIS FILE IS DYNAMICALLY USED FOR THE BACKEND
# TAKE CARE WHEN MODYFING IT ;)
#################################################
# It contains information regarding ow models
# are configured.
# A restart of the backend server is required
# for it to take effect.
#################################################
# Look at the documentation about config files to know more about this
# http://localhost:5000/#/Application/Backend/config_files?id=models_configyml
# https://rex-dri.gitlab.utc.fr/rex-dri/documentation/#/Application/Backend/config_files?id=models_configyml
UserData:
moderation_level: 0
Recommendation:
moderation_level: 0
RecommendationList:
moderation_level: 0
ForTestingModeration:
moderation_level: 1
ForTestingVersioning:
moderation_level: 1
import json
from os.path import join, realpath, dirname
from .utils import load_defaults
current_dir = dirname(realpath(__file__))
with open(join(current_dir, "OBJ_MODERATION_PERMISSIONS.json"), "r") as f:
tmp = json.load(f)
tmp = load_defaults().DEFAULT_OBJ_MODERATION_PERMISSIONS
OBJ_MODERATION_PERMISSIONS = {}
......
from os.path import dirname, join, realpath
from typing import List
import yaml
from dotmap import DotMap
CURRENT_DIR = dirname(realpath(__file__))
def get_yml_file(name):
"""
Helper function to load config files.
"""
with open(join(CURRENT_DIR, name), "r") as f:
return yaml.load(f)
def load_viewsets_config() -> List[DotMap]:
"""
Returns the list of api objects without filtering add default attributes
added to all objects if they are missing.
"""
tmp = get_yml_file("viewsets_config.yml")
api_config = []
for viewset in tmp.keys():
obj = tmp[viewset]
obj["viewset"] = viewset
api_config.append(obj)
# clean api_config (add default arguments)
DEFAULT_SETTINGS = {
"is_api_view": False,
"requires_testing": False,
"viewset_permission": "default",
"api_attr": None,
"api_name": None,
}
for obj in api_config:
for key in DEFAULT_SETTINGS:
if key not in obj:
obj[key] = DEFAULT_SETTINGS[key]
return [DotMap(obj) for obj in api_config]
def load_models_config() -> DotMap:
"""
Load the `models_config.yml` config file and returns it as
a `DotMap`/dict. No transformation whatsoever is applied.
"""
data = get_yml_file("models_config.yml")
return DotMap(data)
def load_defaults() -> DotMap:
"""
Load the `defaults.yml` config file and returns it as
a `DotMap`/dict. No transformation whatsoever is applied.
"""
data = get_yml_file("defaults.yml")
return DotMap(data)
import importlib
from os.path import dirname, join, realpath
from typing import List, Optional, Union, Dict
from typing import List, Optional, Union
from django.conf import settings
import yaml
from dotmap import DotMap
from .obj_moderation_permission import OBJ_MODERATION_PERMISSIONS
from .utils import load_viewsets_config
def load_api_config() -> List[Dict]:
"""
Returns the list of api objects without filtering add default attributes
added to all objects if they are missing.
"""
current_dir = dirname(realpath(__file__))
with open(join(current_dir, "api_config.yml"), "r") as f:
api_config = yaml.load(f)
# clean api_config (add default arguments)
DEFAULT_SETTINGS = {
"is_api_view": False,
"ignore_in_admin": False,
"requires_testing": False,
"moderation_level": 2,
"versionned": False,
"read_only": False,
"viewset_permission": "default",
"model": None,
}
for obj in api_config:
tmp = "enforce_moderation_user_level"
if tmp in obj.keys():
obj[tmp] = OBJ_MODERATION_PERMISSIONS[obj[tmp]]
for key in DEFAULT_SETTINGS:
if key not in obj:
obj[key] = DEFAULT_SETTINGS[key]
return api_config
def get_api_objs(
has_model: Optional[bool],
def get_viewsets_info(
ignore_in_admin: Optional[bool] = None,
versionned: Optional[bool] = None,
requires_testing: Union[None, bool, "smart"] = None,
is_api_view: Optional[bool] = False,
ignore_models: List[str] = list(),
make_imports: bool = True,
) -> List[DotMap]:
"""
Returns a list of DotMap objects corresponding the api config file
with filtering applied to some attributes.
If the parameter is `None` then no filtering is applied to this attribute.
If the parameter is `True` or `False` the object is returned only if it matched.
If the parameter is `True` or `False` the object is returned only if it matches.
There is one exception for the parameter `requires_testing` if it is set to `smart` then
the object is returned only if doesn't require testing or if testing is activated.
make_imports: do we perform the model and viewsets imports ?
"""
out = list()
for entry in load_api_config():
obj = DotMap(entry)
if has_model is not None:
if has_model and obj.model is None:
continue
if not has_model and obj.model is not None:
continue
if versionned is not None:
if versionned and not obj.versionned:
continue
if not versionned and obj.versionned:
continue
for obj in load_viewsets_config():
if ignore_in_admin is not None:
if ignore_in_admin and not obj.ignore_in_admin:
......@@ -105,20 +50,12 @@ def get_api_objs(
if not is_api_view and obj.is_api_view:
continue
if make_imports:
module = importlib.import_module(
"backend_app.models.{}".format(obj.import_location)
)
if obj.model is not None:
if obj.model in ignore_models:
continue
Model = getattr(module, obj.model)
obj.Model = Model
module = importlib.import_module(
"backend_app.models.{}".format(obj.import_location)
)
if obj.viewset is not None:
Viewset = getattr(module, obj.viewset)
obj.Viewset = Viewset
Viewset = getattr(module, obj.viewset)
obj.Viewset = Viewset
out.append(obj)
......
# THIS FILE IS DYNAMICALLY USED FOR THE BACKEND AND THE FRONTEND
# TAKE CARE WHEN MODYFING IT ;)
# model : the model name (may be absent) Model can't be present more than once.
# viewset : the viewset name for the api
# api_end_pont : the main part of the url for making request to the api
# This string will also be used for naming variables in JS !!
# So no weird characters there please...
# versionned: boolean to specify wether this model is versionned or not
# api_attr : to specify some attributes that may be captured
# and used in the viewset
# requires_testing: boolean to tell if this viewset is only availble in
# a testing environment.
# ignore_in_admin: don't register the model in the admin
# Moderation levels are defined as follow :
# 0 : moderation will never be applied
# 1 : moderation will be on if the global settings for moderation is turned on
# 2 : (default for security reasons) moderation will always be on no matter what
# It is to be noted that staff members, dri members and moderators won't be subject to moderation !
#
# When moderation_level > 0, someone may decide to enforce moderation on for the users with a lower
# status in the app. This is called object level moderation !
# staff ⊂ dri ⊂ moderators ⊂ authenficated_user
# For viewset permissions we have the followings
#
# By default, every viewset will have :
# - isAuthentificated : to use the API the client needs to be authentificated
# - noDeleteIfNotStaff : nothing can be deleted except if you are a staff member
#
# Some viewsets may have more presice permissions
# - IsStaff
# - IsStaffOrReadOnly
# - IsDriOrReadOnly
# - IsOwner : (or )
#
# If you have a custom get_queryset function in your viewset, you need to specify an api_name
#################################################
# It contains information regarding
# how viewsets are configured.
# A restart of the backend server and a full
# recompile of the frontend is required
# for changes it to take effect.
#################################################
# Look at the documentation about config files to know more about this
# http://localhost:5000/#/Application/Backend/config_files?id=viewsets_configyml
# https://rex-dri.gitlab.utc.fr/rex-dri/documentation/#/Application/Backend/config_files?id=viewsets_configyml
#####################################################
## Custom Viewsets that doesn't have a model behind
#####################################################
- viewset: AppModerationStatusViewSet
AppModerationStatusViewSet:
api_end_point: serverModerationStatus
import_location: other_viewsets
read_only: true
viewset_permission: ReadOnly
is_api_view: true
#####################
## Standard Viewsets
#####################
- model: Country
viewset: CountryViewSet
CountryViewSet:
import_location: country
api_end_point: countries
viewset_permission: IsStaffOrReadOnly
viewset_permission: IsStaff | ReadOnly
- model: City
viewset: CityViewSet
CityViewSet:
import_location: city
api_end_point: cities
moderation_level: 2
viewset_permission: NoPostIfNotStaff
viewset_permission: IsStaff | NoPost
- model: University
viewset: UniversityViewSet
UniversityViewSet:
import_location: university
api_end_point: universities
moderation_level: 2
viewset_permission: NoPostIfNotStaff
viewset_permission: IsStaff | NoPost
- model: Campus
viewset: CampusViewSet
CampusViewSet:
import_location: campus
api_end_point: campuses
versionned: true
moderation_level: 2
- model: UserData
viewset: UserDataViewSet
UserDataViewSet:
import_location: user
api_end_point: userData
api_name: user-data-detail
moderation_level: 0
viewset_permission: IsOwner
- model: Tag
viewset: TagViewSet
TagViewSet:
import_location: tag
api_end_point: tags
moderation_level: 2
viewset_permission: IsStaffOrReadOnly
viewset_permission: IsStaff | ReadOnly
- model: Currency
viewset: CurrencyViewSet
CurrencyViewSet:
import_location: currency
api_end_point: currencies
moderation_level: 2
viewset_permission: IsStaffOrReadOnly
viewset_permission: IsStaff | ReadOnly
- model: Department
viewset: DepartmentViewSet
DepartmentViewSet:
import_location: other_core
api_end_point: departments
moderation_level: 2
viewset_permission: IsStaffOrReadOnly
viewset_permission: IsStaff | ReadOnly
- model: Specialty
viewset: SpecialtyViewSet
SpecialtyViewSet:
import_location: other_core
api_end_point: specialties
moderation_level: 2
viewset_permission: IsStaffOrReadOnly
viewset_permission: IsStaff | ReadOnly
- model: Offer
viewset: OfferViewSet
OfferViewSet:
import_location: other_core
api_end_point: offers
moderation_level: 2
viewset_permission: IsStaffOrReadOnly
viewset_permission: IsStaff | ReadOnly
- model: CountryTaggedItem
viewset: CountryTaggedItemViewSet
CountryTaggedItemViewSet:
import_location: country
api_end_point: countryTaggedItems
api_attr: (?P<country_id>[a-zA-Z]+)
versionned: true
- model: CountryScholarship
viewset: CountryScholarshipViewSet
CountryScholarshipViewSet:
import_location: country
api_end_point: countryScholarships
api_attr: (?P<country_id>[a-zA-Z]+)
versionned: true
- model: CountryDri
viewset: CountryDriViewSet
CountryDriViewSet:
import_location: country
api_end_point: countryDri
api_attr: (?P<country_id>[a-zA-Z]+)
enforce_moderation_user_level: 'DRI'
viewset_permission: IsDriOrNoPost
versionned: true
viewset_permission: IsStaff | IsDri | NoPost
- model: CityTaggedItem
viewset: CityTaggedItemViewSet
CityTaggedItemViewSet:
import_location: city
api_end_point: cityTaggedItems
api_attr: (?P<city_id>[0-9]+)
versionned: true
- model: UniversityTaggedItem
viewset: UniversityTaggedItemViewSet
UniversityTaggedItemViewSet:
import_location: university
api_end_point: universityTaggedItems
api_attr: (?P<univ_id>[0-9]+)
versionned: true
- model: UniversityScholarship
viewset: UniversityScholarshipViewSet
UniversityScholarshipViewSet:
import_location: university
api_end_point: universityScholarships
api_attr: (?P<univ_id>[0-9]+)
versionned: true
- model: UniversityInfo
viewset: UniversityInfoViewSet
UniversityInfoViewSet:
import_location: university
viewset_permission: NoPostIfNotStaff
viewset_permission: IsStaff | NoPost
api_end_point: universitiesInfo
versionned: true
- model: UniversitySemestersDates
viewset: UniversitySemestersDatesViewSet
UniversitySemestersDatesViewSet:
import_location: university
api_end_point: universitiesSemestersDates
viewset_permission: NoPostIfNotStaff
versionned: true
viewset_permission: IsStaff | NoPost
- model: UniversityDri
viewset: UniversityDriViewSet
UniversityDriViewSet:
import_location: university
api_end_point: universityDri
api_attr: (?P<univ_id>[0-9]+)
enforce_moderation_user_level: 'DRI'
viewset_permission: IsDriOrNoPost
versionned: true
viewset_permission: IsStaff | IsDri | NoPost
- model: CampusTaggedItem
viewset: CampusTaggedItemViewSet
CampusTaggedItemViewSet:
import_location: campus
api_end_point: campusTaggedItems
api_attr: (?P<campus_id>[0-9]+)
versionned: true
- model: null
viewset: MainCampusViewSet
MainCampusViewSet:
import_location: campus
api_end_point: mainCampuses
read_only: true
viewset_permission: ReadOnly
- model: Recommendation
viewset: RecommendationViewSet
RecommendationViewSet:
import_location: user
api_end_point: userRecommendations
moderation_level: 0
- model: RecommendationList
viewset: RecommendationListViewSet
RecommendationListViewSet:
import_location: user
api_end_point: userRecommendationLists
moderation_level: 0
- model: PreviousDeparture
viewset: PreviousDepartureViewSet
PreviousDepartureViewSet:
import_location: user
api_end_point: universitiesPreviousDepartures
read_only: true
viewset_permission: ReadOnly
- model: PreviousDepartureFeedback
viewset: PreviousDepartureFeedbackViewSet
PreviousDepartureFeedbackViewSet:
import_location: user
api_end_point: universitiesPreviousDepartureFeedback
- model: PendingModeration
viewset: PendingModerationViewSet
PendingModerationViewSet:
import_location: abstract.my_model
api_end_point: pendingModeration