Merge branch 'load_data' into 'frontend'

New loading scripts

See merge request chehabfl/outgoing_rex!14

backend/load_data/__init__.py

+from .load_all import load_all  # noqa: F401

backend/load_data/load_all.py

+from .loading_scripts import LoadAdminUser
+from .loading_scripts import LoadCountries
+from .loading_scripts import LoadUniversities
+
+
+def load_all():
+    admin = LoadAdminUser().get()
+    LoadCountries(admin).load()
+    LoadUniversities(admin).load()

backend/load_data/loading_scripts/__init__.py

+from .loadAdminUser import LoadAdminUser  # noqa: F401
+from .loadCountries import LoadCountries  # noqa: F401
+from .loadUniversities import LoadUniversities  # noqa: F401

backend/load_data/loading_scripts/loadAdminUser.py

+from django.contrib.auth.models import User
+
+
+class LoadAdminUser(object):
+    def __init__(self):
+        admins = User.objects.filter(username="admin")
+        if len(admins) > 0:
+            self.admin = admins[0]
+
+        else:
+            self.admin = User.objects.create_superuser(
+                username='admin',
+                email='null@null.fr',
+                password='admin'
+            )
+
+    def get(self):
+        return self.admin

backend/load_data/loading_scripts/loadCountries.py

+from backend.models.location import Country
+import os
+import pandas as pd
+from .loadGeneric import LoadGeneric
+
+
+class LoadCountries(LoadGeneric):
+
+    def __init__(self, admin):
+        self.admin = admin
+
+    def load(self):
+        tmp = os.path.join(os.path.realpath(__file__),
+                           '../../assets/country.csv')
+        country_file_loc = os.path.abspath(tmp)
+        tmp = os.path.join(os.path.realpath(__file__),
+                           '../../assets/alpha-conv-table.csv')
+        conv_alpha_file_loc = os.path.abspath(tmp)
+
+        country_pd = pd.read_csv(country_file_loc, sep=',', header=0,
+                                 dtype=object).fillna('')
+
+        # Need to load the information for converting
+        # Countries alpha-3 code to alpha-2 code
+        data_conv = pd.read_csv(conv_alpha_file_loc, sep=',',
+                                header=0, na_filter=False)
+        conv_alpha = {}
+        for index, row in data_conv.iterrows():
+            conv_alpha[row["alpha-3"]] = row["alpha-2"]
+
+        for index, r in country_pd.iterrows():
+            Iso_3 = str(r['ISO-alpha3 Code'])
+            code_alpha_2 = None
+            if Iso_3 in conv_alpha.keys():
+                code_alpha_2 = conv_alpha[Iso_3]
+            else:
+                print("ignoring country :", Iso_3)
+                continue
+
+            country = Country(
+                name=r["Country or Area"],
+                iso_alpha2_code=code_alpha_2,
+                iso_alpha3_code=Iso_3,
+                region_name=r["Region Name"],
+                region_un_code=r["Region Code"],
+                sub_region_name=r["Sub-region Name"],
+                sub_region_un_code=r["Sub-region Code"],
+                intermediate_region_name=r["Intermediate Region Name"],
+                intermediate_region_un_code=r["Intermediate Region Code"],
+            )
+            country.save()
+            self.add_info(country, self.admin)

backend/load_data/loading_scripts/loadGeneric.py

+from django.utils import timezone
+import reversion
+
+
+class LoadGeneric(object):
+
+    @classmethod
+    def add_info(cls, obj, admin):
+        with reversion.create_revision():
+            obj.moderated_by = admin
+            obj.updated_by = admin
+            obj.moderated_on = timezone.now()
+            obj.updated_on = timezone.now()
+            obj.save()

backend/load_data/loading_scripts/loadUniversities.py

+from backend.models.location import City, Country
+from backend.models.university import University, Campus
+
+import os
+import pandas as pd
+from .loadGeneric import LoadGeneric
+
+
+class LoadUniversities(LoadGeneric):
+
+    def __init__(self, admin):
+        self.admin = admin
+
+    def load(self):
+
+        tmp = os.path.join(os.path.realpath(__file__),
+                           '../../assets/destinations_extracted.csv')
+        destinations_path = os.path.abspath(tmp)
+
+        data = pd.read_csv(destinations_path, sep=',', header=0,
+                           dtype=object)
+
+        for index, row in data.iterrows():
+            univ_name, city_name, country_code, lat, lon = row
+            lat = round(float(lat), 6)
+            lon = round(float(lon), 6)
+
+            country = Country.objects.filter(pk=country_code)[0]
+            city = City(
+                name=city_name,
+                country=country
+            )
+            city.save()
+            self.add_info(city, self.admin)
+
+            univ = University(
+                name=univ_name
+            )
+            univ.save()
+            self.add_info(univ, self.admin)
+
+            main_campus = Campus(
+                is_main_campus=True,
+                name="Campus - " + univ.name,
+                city=city,
+                university=univ,
+                lat=lat,
+                lon=lon
+            )
+            main_campus.save()
+            self.add_info(main_campus, self.admin)

backend/models/university/campus.py

@@ -6,9 +6,6 @@ from django.core.validators import MinValueValidator, MaxValueValidator
 
 
 class Campus(BasicModule):
-    module_icon = "You forget to override the icon in the sub-class : I know"
-    module_name = "Campus principal"
-    module_description = "Campus de l'université."
 
     is_main_campus = models.BooleanField(null=False)
     name = models.CharField(max_length=200, null=True)

backend/tests/test_loading_data.py

+from django.test import TestCase
+from backend.load_data import load_all
+
+
+class ModerationTestCase(TestCase):
+
+    def test_everything_loads(self):
+        try:
+            load_all()
+        except Exception:
+            self.fail("load_all() raised an Exception unexpectedly!")

backend/utils/insert_country.py

-#!/usr/bin/env python3
-
-"""
-Script to insert the country data in the database
-
-IT HAS TO BE RUN INSIDE ./manage.py shell
-"""
-import os
-import argparse
-import pandas as pd
-import requests
-
-if __name__ != "__main__":
-    print(__name__)
-    raise Exception("Script has to be run directly")
-
-# getting args from command line
-parser = argparse.ArgumentParser(
-    description='Country and Region Insertion')
-
-parser.add_argument('--address', '-a', dest='address',
-                    action='store', default='http://127.0.0.1:8000/api',
-                    help='Adresse of the web REST API')
-
-parser.add_argument('--token', '-t', dest='token',
-                    action='store', default='NO_TOKEN_PROVIDED',
-                    help='Permanent token to use')
-
-args = parser.parse_args()
-
-api_address = str(args.address)
-api_token = str(args.token)
-
-tmp = os.path.join(os.path.realpath(__file__), '../../assets/country.csv')
-country_file_loc = os.path.abspath(tmp)
-tmp = os.path.join(os.path.realpath(__file__),
-                   '../../assets/alpha-conv-table.csv')
-conv_alpha_file_loc = os.path.abspath(tmp)
-
-data = pd.read_csv(country_file_loc, sep=',', header=0,
-                   dtype=object).fillna('')
-data = data.drop(["Least Developed Countries (LDC)",
-                  "Land Locked Developing Countries (LLDC)",
-                  "Small Island Developing States (SIDS)",
-                  "Developed / Developing Countries"],
-                 axis=1)
-
-# Need to load the information for converting
-# Countries alpha-3 code to alpha-2 code
-data_conv = pd.read_csv(conv_alpha_file_loc, sep=',',
-                        header=0, na_filter=False)
-conv_alpha_code = {}
-for index, row in data_conv.iterrows():
-    conv_alpha_code[row["alpha-3"]] = row["alpha-2"]
-
-
-def make_post(address, data):
-    h = {'Authorization': 'Token ' + api_token}
-    r = requests.post(address, headers=h, data=data)
-    if r.status_code is 403:
-        raise Exception("Authentification failed")
-    if not r.ok:
-        print(r.content)
-        print(data)
-    return r.ok
-
-
-def save_country(attributes):
-    a = api_address + "/country/"
-    return make_post(a, attributes)
-
-
-file_attr = ["Region Code", "Region Name",
-             "Sub-region Code", "Sub-region Name",
-             "Intermediate Region Code", "Intermediate Region Name",
-             "ISO-alpha3 Code", "Country or Area"]
-
-db_attr = [
-    'region_un_code', 'region_name',
-    'sub_region_un_code', 'sub_region_name',
-    'intermediate_region_un_code', 'intermediate_region_name',
-    'iso_alpha3_code', 'name']
-
-
-for index, row in data.iterrows():
-    country_attributes = {}
-
-    for f_a, db_a in zip(file_attr, db_attr):
-        country_attributes[db_a] = str(row[f_a])
-
-    tmp_c = str(row['ISO-alpha3 Code'])
-    if tmp_c in conv_alpha_code.keys():
-        country_attributes['iso_alpha2_code'] = conv_alpha_code[tmp_c]
-        save_country(country_attributes)
-    else:
-        print("ignoring country :", country_attributes)

backend/utils/insert_universities.py

-#!/usr/bin/env python3
-
-"""
-Script to insert the country data in the database
-
-IT HAS TO BE RUN INSIDE ./manage.py shell
-"""
-import argparse
-import os
-import pandas as pd
-import requests
-import json
-
-if __name__ != "__main__":
-    print(__name__)
-    raise Exception("Script has to be run directly")
-
-# getting args from command line
-parser = argparse.ArgumentParser(
-    description='Country and Region Insertion')
-
-parser.add_argument('--address', '-a', dest='address',
-                    action='store', default='http://127.0.0.1:8000/api',
-                    help='Adresse of the web REST API')
-
-parser.add_argument('--token', '-t', dest='token',
-                    action='store', default='NO_TOKEN_PROVIDED',
-                    help='Permanent token to use')
-
-args = parser.parse_args()
-
-api_address = str(args.address)
-api_token = str(args.token)
-
-
-tmp = os.path.join(os.path.realpath(__file__),
-                   '../../assets/destinations_extracted.csv')
-destinations_path = os.path.abspath(tmp)
-
-data = pd.read_csv(destinations_path, sep=',', header=0,
-                   dtype=object)
-
-
-def make_post(address, data):
-    h = {'Authorization': 'Token ' + api_token}
-    r = requests.post(address, headers=h, data=data)
-    if r.status_code is 403:
-        raise Exception("Authentification failed")
-    print(r.content)
-    return json.loads(r.content)["id"]
-
-
-inserted_city_cc = {}
-
-
-def insert_city(city_name, country):
-    if not city_name + country in inserted_city_cc.keys():
-        address = api_address + "/city/"
-        data = {'country': country, 'name': city_name}
-        inserted_city_cc[city_name + country] = make_post(address, data)
-    return inserted_city_cc[city_name + country]
-
-
-def insert_university(name):
-    address = api_address + "/university/"
-    data = {'name': name}
-    return make_post(address, data)
-
-
-def insert_main_campus(name, city_id, univ_id, lat, lon):
-    address = api_address + "/campus/"
-    data = {'name': "Campus " + name, 'city': city_id, 'lat': lat,
-            'lon': lon, 'is_main_campus': True,
-            'university': univ_id}
-    return make_post(address, data)
-
-
-for index, row in data.iterrows():
-    univ, city, country, lat, lon = row
-    lat = round(float(lat), 6)
-    lon = round(float(lon), 6)
-    city_id = insert_city(city, country)
-    univ_id = insert_university(univ)
-    m_c_id = insert_main_campus(univ, city_id, univ_id, lat, lon)

dev_requirements.txt

 #OutGoing_REX_dev
-pandas # usefull for reading csv but necessary for the app to work
 requests

docs/load.md

 Chargement des données initiales pour l'application
 =====
 
-Pour charger des données dans l'application, tout se passe via l'API du site.
+Pour charger des données de base dans l'application, il suffit d'utiliser les classes `loadCountries`, `loadUniversities`, etc... disponibles dans `backend/load_data/loading_scripts`.
+
+Autrement, pour tout charger d'un coup (et créer un administrateur avec pour identifiant "admin" et pour mot de passe "admin" **PENSER À LE CHANGER EN PRODUCTION...**), il suffit de passer par le shell de Django :
 
-- Créer un superutilisateur : 
 ```bash
-./manage.py createsuperuser
+./manage.py shell
 ```
 
-- Créer un token dans l'administration (pour l'administrateur que vous venez de créer).
-**CE TOKEN DOIT RESTER SECRET**.
-
-- Lancer les scripts pour insérer les données :
-```
-./backend/utils/insert_country.py -t <Token> [-a <server adress>]
-./backend/utils/insert_universities.py-t <Token> [-a <Adresse-de-l'API>]
+puis
+```python
+from backend.load_data import load_all
+load_all()
 ```
+
+Ces scripts sont _unit-testés_ donc ils devraient fonctionner.
\ No newline at end of file

requirements.txt

@@ -17,4 +17,5 @@ pytest-django==3.4.2
 pytest-cov==2.5.1
 pytest-xdist==1.23.0
 flake8
-django-debug-toolbar==1.9.1
\ No newline at end of file
+django-debug-toolbar==1.9.1
+pandas
\ No newline at end of file