Merge branch 'update_back_linting' into 'master'

Update back linting and fix documentation job failing on master

Closes #44

See merge request rex-dri/rex-dri!44

.gitlab-ci.yml

 stages:
   - test
   - linting
-  - doc
+  - documentation
   - deploy
 
 variables:
@@ -21,7 +21,7 @@ testing:
     - make test_backend_server
   artifacts:
     paths:
-      - htmlcov/
+      - backend/htmlcov/
     expire_in: 1 month
   tags:
     - docker
@@ -44,13 +44,13 @@ eslint:
     - docker
 
 pages:
-  stage: doc
+  stage: documentation
   image: floawfloaw/plantuml
   dependencies:
     - testing
   script:
     - mkdir .public
-    - mv htmlcov/ .public/coverage
+    - mv backend/htmlcov/ .public/coverage
     - make documentation
     - mv documentation/ .public/documentation
     - mv .public public

.vscode/settings.json

@@ -14,5 +14,8 @@
         ".pytest_cache": true,
     },
     "cSpell.language": "en,fr-FR,fr",
-    "python.linting.flake8Enabled": true
+    "python.linting.flake8Enabled": true,
+    "python.formatting.provider": "black",
+    "python.linting.pylintEnabled": false,
+    "python.linting.enabled": true
 }
\ No newline at end of file

backend/.flake8

 [flake8]
 # E501 line too long
-ignore = E501
+ignore = E501,W503
 exclude = 
     env/*
     node_modules/*

backend/backend_app/__init__.py

-default_app_config = 'backend_app.apps.BackendAppConfig'
+default_app_config = "backend_app.apps.BackendAppConfig"

backend/backend_app/admin.py

@@ -11,10 +11,12 @@ CLASSIC_MODELS = []
 VERSIONNED_MODELS = []
 
 for model in api_config:
-    if "model" in model and model['model']:
+    if "model" in model and model["model"]:
         model = DotMap(model)
         if (not model.requires_testing) and (not model.ignore_in_admin):
-            module = importlib.import_module("backend_app.models.{}".format(model.import_location))
+            module = importlib.import_module(
+                "backend_app.models.{}".format(model.import_location)
+            )
             if model.versionned:
                 VERSIONNED_MODELS.append(getattr(module, model.model))
             else:
@@ -31,6 +33,5 @@ for model in CLASSIC_MODELS:
 
 for model in VERSIONNED_MODELS:
     admin.site.register(model, CompareVersionAdmin)
-    if (model.get_serializer().Meta.model != model):
-        raise Exception(
-            "Get_serializer configuration incorrect in", str(model))
+    if model.get_serializer().Meta.model != model:
+        raise Exception("Get_serializer configuration incorrect in", str(model))

backend/backend_app/apps.py

@@ -2,9 +2,9 @@ from django.apps import AppConfig
 
 
 class BackendAppConfig(AppConfig):
-    name = 'backend_app'
+    name = "backend_app"
 
     def ready(self):
-        import backend_app.signals.__create_user_modules_post_create  # noqa: F401
-        import backend_app.signals.__squash_revision_by_user  # noqa: F401
-        import backend_app.signals.__create_univ_modules_post_save  # noqa: F401
+        import backend_app.signals.__create_user_modules_post_create  # noqa:F401
+        import backend_app.signals.__squash_revision_by_user  # noqa:F401
+        import backend_app.signals.__create_univ_modules_post_save  # noqa:F401

backend/backend_app/custom/DictModeViewSet.py

@@ -6,12 +6,14 @@ class DictModeViewSet(viewsets.ModelViewSet):
     ViewSet that renders data as dict with keys corresponding to the model
     primary key. Instead of list.
     """
+
     BYPASS_DICT_MODE = False
     LIST_SHOULD_BE_DETAIL = False
 
     def list(self, request, *args, **kwargs):
         response = super(viewsets.ModelViewSet, self).list(  # pylint: disable=E1003
-            request, *args, **kwargs)  # call the original 'list'
+            request, *args, **kwargs
+        )  # call the original 'list'
 
         if self.LIST_SHOULD_BE_DETAIL:
             if len(response.data) == 0:
@@ -20,12 +22,11 @@ class DictModeViewSet(viewsets.ModelViewSet):
                 response.data = response.data[0]
             else:
                 raise Exception(
-                    "There should be no more than one element here check your queryset !")
+                    "There should be no more than one element here check your queryset !"
+                )
 
         elif not self.BYPASS_DICT_MODE:
             pk_attr_name = self.serializer_class.Meta.model._meta.pk.name
-            response.data = {
-                d[pk_attr_name]: d for d in response.data
-            }
+            response.data = {d[pk_attr_name]: d for d in response.data}
 
         return response

backend/backend_app/custom/__init__.py

-from .DictModeViewSet import DictModeViewSet  # noqa: F401
-from .mySerializerWithJSON import MySerializerWithJSON  # noqa: F401
+from .DictModeViewSet import DictModeViewSet
+from .mySerializerWithJSON import MySerializerWithJSON
+
+__all__ = ["DictModeViewSet", "MySerializerWithJSON"]

backend/backend_app/custom/mySerializerWithJSON.py

@@ -9,4 +9,5 @@ class MySerializerWithJSON(serializers.ModelSerializer):
     """
     Simple class to add support for custom JSONField support
     """
+
     serializer_field_mapping = field_mapping

backend/backend_app/fields.py

@@ -17,10 +17,7 @@ class JSONField(models.TextField):
         try:
             return json.loads(value)
         except json.decoder.JSONDecodeError as e:
-            raise ValidationError(
-                message=str(e),
-                code="json_error"
-            )
+            raise ValidationError(message=str(e), code="json_error")
 
     def to_json(self, value):
         """ convert python dictionary to json string """

backend/backend_app/load_data/__init__.py

-from .load_all import load_all  # noqa: F401
+from .load_all import load_all
+
+__all__ = ["load_all"]

backend/backend_app/load_data/assets/extract_universities.py

@@ -11,19 +11,20 @@ import time
 from geopy.geocoders import Nominatim
 import reverse_geocoder as rg
 
-tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations.csv')
+tmp = os.path.join(os.path.realpath(__file__), "../../assets/destinations.csv")
 destinations_path = os.path.abspath(tmp)
 
-tmp = os.path.join(os.path.realpath(__file__),
-                   '../../assets/destinations_extracted.csv')
+tmp = os.path.join(
+    os.path.realpath(__file__), "../../assets/destinations_extracted.csv"
+)
 destinations_extracted_path = os.path.abspath(tmp)
 
 if not os.path.isfile(destinations_path):
     print(destinations_path)
     raise Exception("Missing file containing country data")
 
-with open(destinations_path, 'rt') as input:
-    with open(destinations_extracted_path, 'w') as output:
+with open(destinations_path, "rt") as input:
+    with open(destinations_extracted_path, "w") as output:
         print("ini")
         reader = csv.reader(input)
         spamwriter = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
@@ -34,7 +35,7 @@ with open(destinations_path, 'rt') as input:
         for row in reader:
             # handle the header
             if i == 0:
-                header = ['university', 'city', 'country', 'lat', 'lon']
+                header = ["university", "city", "country", "lat", "lon"]
                 spamwriter.writerow(header)
                 i += 1
             else:
@@ -50,7 +51,13 @@ with open(destinations_path, 'rt') as input:
                 if location is not None:
                     coord = (location.latitude, location.longitude)
                     res = rg.search(coord, verbose=False)
-                    line = [row[2], row[1], res[0]['cc'], location.latitude, location.longitude]
+                    line = [
+                        row[2],
+                        row[1],
+                        res[0]["cc"],
+                        location.latitude,
+                        location.longitude,
+                    ]
                     print(line)
                     spamwriter.writerow(line)
                 else:

backend/backend_app/load_data/loading_scripts/__init__.py

-from .loadGroups import LoadGroups  # noqa: F401
-from .loadAdminUser import LoadAdminUser  # noqa: F401
-from .loadCountries import LoadCountries  # noqa: F401
-from .loadUniversities import LoadUniversities  # noqa: F401
-from .loadTags import LoadTags  # noqa: F401
-from .loadCurrencies import LoadCurrencies  # noqa: F401
-from .loadUniversityEx import LoadUniversityEx  # noqa: F401
+from .loadGroups import LoadGroups
+from .loadAdminUser import LoadAdminUser
+from .loadCountries import LoadCountries
+from .loadUniversities import LoadUniversities
+from .loadTags import LoadTags
+from .loadCurrencies import LoadCurrencies
+from .loadUniversityEx import LoadUniversityEx
+
+__all__ = [
+    "LoadGroups",
+    "LoadAdminUser",
+    "LoadCountries",
+    "LoadUniversities",
+    "LoadTags",
+    "LoadCurrencies",
+    "LoadUniversityEx",
+]

backend/backend_app/load_data/loading_scripts/loadAdminUser.py

@@ -13,9 +13,7 @@ class LoadAdminUser(object):
             self.admin = user[0]
         else:
             User.objects.create_superuser(
-                username="admin",
-                email='null@null.fr',
-                password='admin',
+                username="admin", email="null@null.fr", password="admin"
             )
             self.admin = User.objects.filter(username="admin")[0]
 

backend/backend_app/load_data/loading_scripts/loadCountries.py

@@ -5,31 +5,30 @@ from .loadGeneric import LoadGeneric
 
 
 class LoadCountries(LoadGeneric):
-
     def __init__(self, admin):
         self.admin = admin
 
     def load(self):
-        tmp = os.path.join(os.path.realpath(__file__),
-                           '../../assets/country.csv')
+        tmp = os.path.join(os.path.realpath(__file__), "../../assets/country.csv")
         country_file_loc = os.path.abspath(tmp)
-        tmp = os.path.join(os.path.realpath(__file__),
-                           '../../assets/alpha-conv-table.csv')
+        tmp = os.path.join(
+            os.path.realpath(__file__), "../../assets/alpha-conv-table.csv"
+        )
         conv_alpha_file_loc = os.path.abspath(tmp)
 
-        country_pd = pd.read_csv(country_file_loc, sep=',', header=0,
-                                 dtype=object).fillna('')
+        country_pd = pd.read_csv(
+            country_file_loc, sep=",", header=0, dtype=object
+        ).fillna("")
 
         # Need to load the information for converting
         # Countries alpha-3 code to alpha-2 code
-        data_conv = pd.read_csv(conv_alpha_file_loc, sep=',',
-                                header=0, na_filter=False)
+        data_conv = pd.read_csv(conv_alpha_file_loc, sep=",", header=0, na_filter=False)
         conv_alpha = {}
         for index, row in data_conv.iterrows():
             conv_alpha[row["alpha-3"]] = row["alpha-2"]
 
         for index, r in country_pd.iterrows():
-            Iso_3 = str(r['ISO-alpha3 Code'])
+            Iso_3 = str(r["ISO-alpha3 Code"])
             code_alpha_2 = None
             if Iso_3 in conv_alpha.keys():
                 code_alpha_2 = conv_alpha[Iso_3]

backend/backend_app/load_data/loading_scripts/loadCurrencies.py

@@ -6,13 +6,11 @@ from decimal import Decimal
 
 
 class LoadCurrencies(LoadGeneric):
-
     def __init__(self, admin):
         self.admin = admin
 
     def load(self):
-        tmp = os.path.join(os.path.realpath(__file__),
-                           '../../assets/currencies.csv')
+        tmp = os.path.join(os.path.realpath(__file__), "../../assets/currencies.csv")
         currencies_file_loc = os.path.abspath(tmp)
 
         with open(currencies_file_loc) as csvfile:
@@ -23,7 +21,7 @@ class LoadCurrencies(LoadGeneric):
                     code=r[0],
                     name=r[1],
                     symbol="",
-                    one_EUR_in_this_currency=Decimal(r[2])
+                    one_EUR_in_this_currency=Decimal(r[2]),
                 )
                 currency.save()
                 self.add_info_and_save(currency, self.admin)

backend/backend_app/load_data/loading_scripts/loadGeneric.py

@@ -3,7 +3,6 @@ import reversion
 
 
 class LoadGeneric(object):
-
     @classmethod
     def add_info_and_save(cls, obj, admin):
         with reversion.create_revision():

backend/backend_app/load_data/loading_scripts/loadGroups.py

@@ -3,5 +3,5 @@ from django.contrib.auth.models import Group
 
 class LoadGroups(object):
     def __init__(self):
-        Group.objects.get_or_create(name='Moderators')
-        Group.objects.get_or_create(name='DRI')
+        Group.objects.get_or_create(name="Moderators")
+        Group.objects.get_or_create(name="DRI")

backend/backend_app/load_data/loading_scripts/loadTags.py

@@ -6,18 +6,16 @@ from .loadGeneric import LoadGeneric
 
 
 class LoadTags(LoadGeneric):
-
     def __init__(self, admin):
         self.admin = admin
 
     def load(self):
 
-        tmp = os.path.join(os.path.realpath(__file__),
-                           '../../assets/tags.json')
+        tmp = os.path.join(os.path.realpath(__file__), "../../assets/tags.json")
         tags_path = os.path.abspath(tmp)
         with open(tags_path) as f:
             tags = json.load(f)
             for tag in tags:
-                t = Tag(name=tag['name'], config=tag['config'])
+                t = Tag(name=tag["name"], config=tag["config"])
                 t.save()
                 self.add_info_and_save(t, self.admin)

backend/backend_app/load_data/loading_scripts/loadUniversities.py

@@ -9,29 +9,29 @@ from .loadGeneric import LoadGeneric
 
 
 class LoadUniversities(LoadGeneric):
-
     def __init__(self, admin):
         self.admin = admin
 
     def load(self):
 
-        tmp = os.path.join(os.path.realpath(__file__),
-                           '../../assets/destinations_extracted.csv')
+        tmp = os.path.join(
+            os.path.realpath(__file__), "../../assets/destinations_extracted.csv"
+        )
         destinations_path = os.path.abspath(tmp)
 
-        data = pd.read_csv(destinations_path, sep=',', header=0,
-                           dtype=object).fillna('')
+        data = pd.read_csv(destinations_path, sep=",", header=0, dtype=object).fillna(
+            ""
+        )
 
         for index, row in data.iterrows():
-            utc_id, univ_name, city_name, country_code, lat, lon, acronym, website, logo = row
+            utc_id, univ_name, city_name, country_code, lat, lon, acronym, website, logo = (
+                row
+            )
             lat = round(float(lat), 6)
             lon = round(float(lon), 6)
 
             country = Country.objects.get(pk=country_code)
-            city = City(
-                name=city_name,
-                country=country
-            )
+            city = City(name=city_name, country=country)
             city.save()
             self.add_info_and_save(city, self.admin)
 
@@ -41,8 +41,8 @@ class LoadUniversities(LoadGeneric):
                     "name": univ_name,
                     "acronym": acronym,
                     "website": website,
-                    "logo": logo
-                }
+                    "logo": logo,
+                },
             )[0]
             self.add_info_and_save(univ, self.admin)
 
@@ -52,7 +52,7 @@ class LoadUniversities(LoadGeneric):
                 city=city,
                 university=univ,
                 lat=lat,
-                lon=lon
+                lon=lon,
             )
             main_campus.save()
             self.add_info_and_save(main_campus, self.admin)