Merge branch 'enhancement/backendDynamicImports#46' into 'master'

Enhancement/backend dynamic imports #46

Closes #46

See merge request rex-dri/rex-dri!63

backend/backend_app/admin.py

 from django.contrib import admin
-from dotmap import DotMap
-from reversion_compare.admin import CompareVersionAdmin
-
-from shared import get_api_config
-import importlib
-
-api_config = get_api_config()
-
-# models that are versionned in the app
-VERSIONNED_MODELS = []
-# Other models, ie not versionned
-CLASSIC_MODELS = []
 
+from reversion_compare.admin import CompareVersionAdmin
+from shared import get_api_objs
+
+VERSIONNED_MODELS = map(
+    lambda obj: obj.Model,
+    get_api_objs(
+        has_model=True, ignore_in_admin=False, versionned=True, requires_testing=False
+    ),
+)
+CLASSIC_MODELS = map(
+    lambda obj: obj.Model,
+    get_api_objs(
+        has_model=True, ignore_in_admin=False, versionned=False, requires_testing=False
+    ),
+)
 
-# Go through the API configuraion
-for entry in api_config:
-    if "is_api_view" in entry and entry["is_api_view"]:
-        continue
-    if "model" in entry and entry["model"]:
-        model_obj = DotMap(entry)
-        if (not model_obj.requires_testing) and (not model_obj.ignore_in_admin):
-            # Import the model
-            module = importlib.import_module(
-                "backend_app.models.{}".format(model_obj.import_location)
-            )
-            # Add it to the correct list of models
-            if model_obj.versionned:
-                VERSIONNED_MODELS.append(getattr(module, model_obj.model))
-            else:
-                CLASSIC_MODELS.append(getattr(module, model_obj.model))
 
 #######
-# Some dynamic checks
+# Register the models and perform some dynamic checks
 #######
 
 for Model in CLASSIC_MODELS:
@@ -40,7 +27,10 @@ for Model in CLASSIC_MODELS:
     try:
         # Check that it doesn't have the get_serializer method
         Model.get_serializer()
-        raise Exception("A 'CLASSIC MODEL' SHOULDN'T have the get_serializer method")
+        raise Exception(
+            "A 'CLASSIC MODEL' SHOULDN'T have the "
+            "get_serializer method, {}".format(Model)
+        )
     except AttributeError:
         pass
 

backend/backend_app/other_viewsets.py → backend/backend_app/models/other_viewsets.py

@@ -9,6 +9,7 @@ from shared import OBJ_MODERATION_PERMISSIONS
 
 class AppModerationStatusViewSet(APIView):
     """
+    Viewset to know what is the app moderation status
     """
 
     permission_classes = get_viewset_permissions("AppModerationStatusViewSet")

backend/backend_app/models/user/userData.py

+from django.contrib.auth.models import User
 from django.db import models
-from rest_framework import serializers
-from backend_app.models.university import University
+
 from backend_app.fields import JSONField
 from backend_app.models.abstract.my_model import (
     MyModel,
     MyModelSerializer,
     MyModelViewSet,
 )
-from django.contrib.auth.models import User
-from backend_app.utils import get_viewset_permissions, get_model_config, get_user_level
+from backend_app.models.university import University
 from backend_app.permissions.__list_user_post_permission import (
     list_user_post_permission,
 )
+from backend_app.utils import get_model_config, get_user_level, get_viewset_permissions
+from rest_framework import serializers
 
 
 class UserData(MyModel):

backend/backend_app/permissions/__list_user_post_permission.py

-from django.conf import settings
-from dotmap import DotMap
+from typing import List
 
-from shared import get_api_config
-import importlib
+from django.contrib.auth.models import User
 
-api_config = get_api_config()
+from shared import get_api_objs
 
 ALL_VIEWSETS = {}
 
-for model in api_config:
-    model = DotMap(model)
-    if "is_api_view" in model and model.is_api_view:
-        continue
-    if not model.requires_testing:
-        if model.viewset != "UserDataViewSet":
-            module = importlib.import_module(
-                "backend_app.models.{}".format(model.import_location)
-            )
-            ALL_VIEWSETS[model.viewset] = getattr(module, model.viewset)
-
-if settings.TESTING:
-    for model in api_config:
-        model = DotMap(model)
-        if "is_api_view" in model and model.is_api_view:
-            continue
-        if model.requires_testing:
-            if model.viewset != "UserDataViewSet":
-                module = importlib.import_module(
-                    "backend_app.models.{}".format(model.import_location)
-                )
-                ALL_VIEWSETS[model.viewset] = getattr(module, model.viewset)
+for api_obj in get_api_objs(
+    has_model=None,
+    requires_testing=False,
+    is_api_view=False,
+    ignore_models=["UserData"],
+):
+    ALL_VIEWSETS[api_obj.viewset] = api_obj.Viewset
+
+for api_obj in get_api_objs(
+    has_model=None, requires_testing=True, is_api_view=False, ignore_models=["UserData"]
+):
+    ALL_VIEWSETS[api_obj.viewset] = api_obj.Viewset
 
 
 class Request(object):
@@ -38,7 +26,11 @@ class Request(object):
         self.method = method
 
 
-def list_user_post_permission(user):
+def list_user_post_permission(user: User) -> List[str]:
+    """
+    Function the list the viewset to which a user can submit a post request to.
+    """
+
     viewsets_user_can_post = []
     request = Request(user, "POST")
     for viewset_name in ALL_VIEWSETS:

backend/backend_app/urls.py

-from django.conf import settings
 from django.conf.urls import include, url
 from rest_framework import routers
 from rest_framework.documentation import include_docs_urls
 
 from backend_app.permissions import DEFAULT_VIEWSET_PERMISSIONS
-from shared import get_api_config
+from shared import get_api_objs
 
-from dotmap import DotMap
-from .other_viewsets import AppModerationStatusViewSet
-
-import importlib
 
 urlpatterns = [url(r"^api-docs/", include_docs_urls(title="Outgoing API"))]
 router = routers.DefaultRouter()
 
-ALL_MODELS = []
-ALL_VIEWSETS = []
+ALL_MODELS = map(
+    lambda obj: obj.Model,
+    get_api_objs(has_model=True, ignore_in_admin=False, requires_testing="smart"),
+)
 
-# Automatically load models and viewset based on API config file
-api_config = get_api_config()
+ALL_VIEWSETS = []
 
-for entry in api_config:
-    model_obj = DotMap(entry)
-    if "is_api_view" in model_obj and model_obj.is_api_view:
-        continue
-    if (not model_obj.requires_testing) or (
-        settings.TESTING and model_obj.requires_testing
-    ):
-        module = importlib.import_module(
-            "backend_app.models.{}".format(model_obj.import_location)
-        )
-        Viewset = getattr(module, model_obj.viewset)
-        ALL_VIEWSETS.append(Viewset)
 
-        if model_obj.model is not None and not model_obj.ignore_in_admin:
-            ALL_MODELS.append(getattr(module, model_obj.model))
+for api_obj in get_api_objs(
+    has_model=None, requires_testing="smart", is_api_view=False
+):
+    ALL_VIEWSETS.append(api_obj.Viewset)
 
-        # Creating the correct router entry
-        str_url = model_obj.api_end_point
-        if "api_attr" in model_obj:
-            str_url += "/{}".format(model_obj.api_attr)
-        if "api_name" in model_obj:
-            router.register(str_url, Viewset, model_obj.api_name)
-        else:
-            router.register(str_url, Viewset)
+    # Creating the correct router entry
+    str_url = api_obj.api_end_point
+    if "api_attr" in api_obj:
+        str_url += "/{}".format(api_obj.api_attr)
+    if "api_name" in api_obj:
+        router.register(str_url, api_obj.Viewset, api_obj.api_name)
+    else:
+        router.register(str_url, api_obj.Viewset)
 
 # Add all the endpoints for the base api
-urlpatterns += [
-    url(r"^api/", include(router.urls)),
-    url(r"^api/serverModerationStatus/", AppModerationStatusViewSet.as_view()),
-]
+urlpatterns.append(url(r"^api/", include(router.urls)))
+
+for api_obj in get_api_objs(has_model=None, requires_testing="smart", is_api_view=True):
+    urlpatterns.append(
+        url(r"^api/{}/".format(api_obj.api_end_point), api_obj.Viewset.as_view())
+    )
+
 
 #######
 # Models and Viewset checks

backend/backend_app/utils/__does_user_have_moderation_rights.py

 from backend_app.utils import is_member
+from django.contrib.auth.models import User
 
 
-def does_user_have_moderation_rights(user):
+def does_user_have_moderation_rights(user: User) -> bool:
     """
+    Function to know if a user is staff or member of DRI or member of the moderator group.
+
     TODO unit test
     """
     return user.is_staff or is_member("DRI", user) or is_member("Moderators", user)

backend/backend_app/utils/__find_api_end_point_for_viewset.py

-from shared import get_api_config
+from shared import get_api_objs
 
 
-def find_api_end_point_for_viewset(viewset_name):
+def find_api_end_point_for_viewset(viewset_name: str) -> str:
+    """
+    Gets the api endpoint associated with a viewset
+    """
 
-    api_config = get_api_config()
-    for obj in api_config:
-        if obj["viewset"] == viewset_name:
-            return obj["api_end_point"]
+    for obj in get_api_objs(has_model=None, make_imports=False):
+        if obj.viewset == viewset_name:
+            return obj.api_end_point
 
     return None

backend/backend_app/utils/__get_model_config.py

-from shared import get_api_config
+from shared import get_api_objs
 
 
-def get_model_config(model):
-    api_config = get_api_config()
-
-    for obj in api_config:
-        if "is_api_view" in obj and obj["is_api_view"]:
-            continue
-        if obj["model"] == model:
-            tmp = {
-                "moderation_level": obj["moderation_level"],
+def get_model_config(model: str) -> dict:
+    """
+    Returns the configuraiton of the model
+    """
+    for obj in get_api_objs(has_model=True, is_api_view=False, make_imports=False):
+        if obj.model == model:
+            out = {
+                "moderation_level": obj.moderation_level,
                 "model": model,
-                "read_only": obj["read_only"],
+                "read_only": obj.read_only,
             }
             key = "enforce_moderation_user_level"
             if key in obj.keys():
-                tmp[key] = obj[key]
-            return tmp
+                out[key] = obj[key]
+            return out
 
-    raise Exception("Model not found in API configuration, cannot process !")
+    raise Exception(
+        "Model {} not found in API configuration, cannot process !".format(model)
+    )

backend/backend_app/utils/__get_user_level.py

 from .__is_member import is_member
+from django.contrib.auth.models import User
 from shared import OBJ_MODERATION_PERMISSIONS
 
 
-def get_user_level(user) -> int:
+def get_user_level(user: User) -> int:
     """
+    Returns the user level as int.
     TODO unit test
     """
     if user.is_staff:

backend/backend_app/utils/__get_viewset_permissions.py

@@ -8,14 +8,17 @@ from backend_app.permissions import (
 )
 from rest_framework.permissions import IsAdminUser
 from backend_app.permissions import DEFAULT_VIEWSET_PERMISSIONS
-from shared import get_api_config
+from shared import get_api_objs
 
 
-def get_viewset_permissions(viewset):
-    api_config = get_api_config()
-    for obj in api_config:
-        if obj["viewset"] == viewset:
-            custom_permission = obj["viewset_permission"]
+def get_viewset_permissions(viewset: str) -> object:
+    """
+    Returns the permissions associated with  the viewset as configured in the config file.
+    """
+
+    for obj in get_api_objs(has_model=None, make_imports=False, is_api_view=None):
+        if obj.viewset == viewset:
+            custom_permission = obj.viewset_permission
             if custom_permission == "IsOwner":
                 permission = (IsOwner,)
             elif custom_permission == "IsStaffOrReadOnly":
@@ -33,8 +36,10 @@ def get_viewset_permissions(viewset):
             else:
                 raise Exception("Permission not supported ! Dev what did you do ?")
 
-            if obj["read_only"]:
+            if obj.read_only:
                 permission += (ReadOnly,)
             return DEFAULT_VIEWSET_PERMISSIONS + permission
 
-    raise Exception("Viewset not found in API configuraiton, cannot process !")
+    raise Exception(
+        "Viewset {} not found in API configuraiton, cannot proceed !".format(viewset)
+    )

backend/backend_app/utils/__is_member.py

-def is_member(group_name, user):
+from django.contrib.auth.models import User
+
+
+def is_member(group_name: str, user: User) -> bool:
     """
     Function to know if a user is part of a specific group.
 

shared/__init__.py

-from .get_api_config import get_api_config
+from .get_api_config import get_api_objs
 from .obj_moderation_permission import (
     DEFAULT_OBJ_MODERATION_LV,
     OBJ_MODERATION_PERMISSIONS,
 )
 
-__all__ = ["get_api_config", "DEFAULT_OBJ_MODERATION_LV", "OBJ_MODERATION_PERMISSIONS"]
+__all__ = ["DEFAULT_OBJ_MODERATION_LV", "OBJ_MODERATION_PERMISSIONS", "get_api_objs"]

shared/api_config.yml

 # THIS FILE IS DYNAMICALLY USED FOR THE BACKEND AND THE FRONTEND
 # TAKE CARE WHEN MODYFING IT ;)
 
-# model : the model name (may be null) Model can't be present more than once.
+# model : the model name (may be absent) Model can't be present more than once.
 # viewset : the viewset name for the api
 # api_end_pont : the main part of the url for making request to the api
 #  This string will also be used for naming variables in JS !!
@@ -27,7 +27,7 @@
 #
 # By default, every viewset will have :
 #     - isAuthentificated : to use the API the client needs to be authentificated
-#     - noDeleteIsNotStaff : nothing can be deleted except if you are a staff member
+#     - noDeleteIfNotStaff : nothing can be deleted except if you are a staff member
 #
 # Some viewsets may have more presice permissions
 #     - IsStaff
@@ -36,11 +36,20 @@
 #     - IsOwner : (or )
 #
 
+
+#####################################################
+## Custom Viewsets that doesn't have a model behind
+#####################################################
 - viewset: AppModerationStatusViewSet
   api_end_point: serverModerationStatus
+  import_location: other_viewsets
   read_only: true
   is_api_view: true
 
+#####################
+## Standard Viewsets
+#####################
+
 - model: Country
   viewset: CountryViewSet
   import_location: country

shared/get_api_config.py

+import importlib
+from os.path import dirname, join, realpath
+from typing import List, Optional, Union, Dict
+
+from django.conf import settings
+
 import yaml
-from os.path import join, realpath, dirname
+from dotmap import DotMap
+
 from .obj_moderation_permission import OBJ_MODERATION_PERMISSIONS
 
 
-def get_api_config():
+def load_api_config() -> List[Dict]:
+    """
+    Returns the list of api objects without filtering add default attributes
+    added to all objects if they are missing.
+    """
+
     current_dir = dirname(realpath(__file__))
     with open(join(current_dir, "api_config.yml"), "r") as f:
         api_config = yaml.load(f)
 
     # clean api_config (add default arguments)
     DEFAULT_SETTINGS = {
+        "is_api_view": False,
         "ignore_in_admin": False,
         "requires_testing": False,
         "moderation_level": 2,
         "versionned": False,
         "read_only": False,
         "viewset_permission": "default",
+        "model": None,
     }
 
     for obj in api_config:
@@ -27,3 +41,85 @@ def get_api_config():
                 obj[key] = DEFAULT_SETTINGS[key]
 
     return api_config
+
+
+def get_api_objs(
+    has_model: Optional[bool],
+    ignore_in_admin: Optional[bool] = None,
+    versionned: Optional[bool] = None,
+    requires_testing: Union[None, bool, "smart"] = None,
+    is_api_view: Optional[bool] = False,
+    ignore_models: List[str] = list(),
+    make_imports: bool = True,
+) -> List[DotMap]:
+    """
+    Returns a list of DotMap objects corresponding the api config file
+    with filtering applied to some attributes.
+    If the parameter is `None` then no filtering is applied to this attribute.
+    If the parameter is `True` or `False` the object is returned only if it matched.
+
+    There is one exception for the parameter `requires_testing` if it is set to `smart` then
+    the object is returned only if doesn't require testing or if testing is activated.
+
+    make_imports: do we perform the model and viewsets imports ?
+    """
+
+    out = list()
+
+    for entry in load_api_config():
+        obj = DotMap(entry)
+
+        if has_model is not None:
+            if has_model and obj.model is None:
+                continue
+            if not has_model and obj.model is not None:
+                continue
+
+        if versionned is not None:
+            if versionned and not obj.versionned:
+                continue
+            if not versionned and obj.versionned:
+                continue
+
+        if ignore_in_admin is not None:
+            if ignore_in_admin and not obj.ignore_in_admin:
+                continue
+            if not ignore_in_admin and obj.ignore_in_admin:
+                continue
+
+        if requires_testing is not None:
+            if requires_testing == "smart":
+                if obj.requires_testing and not settings.TESTING:
+                    continue
+            else:
+                if requires_testing and not settings.TESTING:
+                    continue
+                if requires_testing and not obj.requires_testing:
+                    continue
+                if not requires_testing and obj.requires_testing:
+                    continue
+
+        if is_api_view is not None:
+            if is_api_view and not obj.is_api_view:
+                continue
+            if not is_api_view and obj.is_api_view:
+                continue
+
+        if make_imports:
+            module = importlib.import_module(
+                "backend_app.models.{}".format(obj.import_location)
+            )
+
+            if obj.model is not None:
+                if obj.model in ignore_models:
+                    continue
+                Model = getattr(module, obj.model)
+                obj.Model = Model
+
+            if obj.viewset is not None:
+                Viewset = getattr(module, obj.viewset)
+                obj.Viewset = Viewset
+
+        out.append(obj)
+
+    return out