Commit 54cd053f authored by Florent Chehab's avatar Florent Chehab

Merge branch 'load_data' into 'frontend'

New loading scripts

See merge request chehabfl/outgoing_rex!14
parents 7db3e41d e0eac03d
Pipeline #26545 passed with stages
in 2 minutes and 52 seconds
from .load_all import load_all # noqa: F401
from .loading_scripts import LoadAdminUser
from .loading_scripts import LoadCountries
from .loading_scripts import LoadUniversities
def load_all():
admin = LoadAdminUser().get()
LoadCountries(admin).load()
LoadUniversities(admin).load()
from .loadAdminUser import LoadAdminUser # noqa: F401
from .loadCountries import LoadCountries # noqa: F401
from .loadUniversities import LoadUniversities # noqa: F401
from django.contrib.auth.models import User
class LoadAdminUser(object):
def __init__(self):
admins = User.objects.filter(username="admin")
if len(admins) > 0:
self.admin = admins[0]
else:
self.admin = User.objects.create_superuser(
username='admin',
email='null@null.fr',
password='admin'
)
def get(self):
return self.admin
from backend.models.location import Country
import os
import pandas as pd
from .loadGeneric import LoadGeneric
class LoadCountries(LoadGeneric):
def __init__(self, admin):
self.admin = admin
def load(self):
tmp = os.path.join(os.path.realpath(__file__),
'../../assets/country.csv')
country_file_loc = os.path.abspath(tmp)
tmp = os.path.join(os.path.realpath(__file__),
'../../assets/alpha-conv-table.csv')
conv_alpha_file_loc = os.path.abspath(tmp)
country_pd = pd.read_csv(country_file_loc, sep=',', header=0,
dtype=object).fillna('')
# Need to load the information for converting
# Countries alpha-3 code to alpha-2 code
data_conv = pd.read_csv(conv_alpha_file_loc, sep=',',
header=0, na_filter=False)
conv_alpha = {}
for index, row in data_conv.iterrows():
conv_alpha[row["alpha-3"]] = row["alpha-2"]
for index, r in country_pd.iterrows():
Iso_3 = str(r['ISO-alpha3 Code'])
code_alpha_2 = None
if Iso_3 in conv_alpha.keys():
code_alpha_2 = conv_alpha[Iso_3]
else:
print("ignoring country :", Iso_3)
continue
country = Country(
name=r["Country or Area"],
iso_alpha2_code=code_alpha_2,
iso_alpha3_code=Iso_3,
region_name=r["Region Name"],
region_un_code=r["Region Code"],
sub_region_name=r["Sub-region Name"],
sub_region_un_code=r["Sub-region Code"],
intermediate_region_name=r["Intermediate Region Name"],
intermediate_region_un_code=r["Intermediate Region Code"],
)
country.save()
self.add_info(country, self.admin)
from django.utils import timezone
import reversion
class LoadGeneric(object):
@classmethod
def add_info(cls, obj, admin):
with reversion.create_revision():
obj.moderated_by = admin
obj.updated_by = admin
obj.moderated_on = timezone.now()
obj.updated_on = timezone.now()
obj.save()
from backend.models.location import City, Country
from backend.models.university import University, Campus
import os
import pandas as pd
from .loadGeneric import LoadGeneric
class LoadUniversities(LoadGeneric):
def __init__(self, admin):
self.admin = admin
def load(self):
tmp = os.path.join(os.path.realpath(__file__),
'../../assets/destinations_extracted.csv')
destinations_path = os.path.abspath(tmp)
data = pd.read_csv(destinations_path, sep=',', header=0,
dtype=object)
for index, row in data.iterrows():
univ_name, city_name, country_code, lat, lon = row
lat = round(float(lat), 6)
lon = round(float(lon), 6)
country = Country.objects.filter(pk=country_code)[0]
city = City(
name=city_name,
country=country
)
city.save()
self.add_info(city, self.admin)
univ = University(
name=univ_name
)
univ.save()
self.add_info(univ, self.admin)
main_campus = Campus(
is_main_campus=True,
name="Campus - " + univ.name,
city=city,
university=univ,
lat=lat,
lon=lon
)
main_campus.save()
self.add_info(main_campus, self.admin)
......@@ -6,9 +6,6 @@ from django.core.validators import MinValueValidator, MaxValueValidator
class Campus(BasicModule):
module_icon = "You forget to override the icon in the sub-class : I know"
module_name = "Campus principal"
module_description = "Campus de l'université."
is_main_campus = models.BooleanField(null=False)
name = models.CharField(max_length=200, null=True)
......
from django.test import TestCase
from backend.load_data import load_all
class ModerationTestCase(TestCase):
def test_everything_loads(self):
try:
load_all()
except Exception:
self.fail("load_all() raised an Exception unexpectedly!")
#!/usr/bin/env python3
"""
Script to insert the country data in the database
IT HAS TO BE RUN INSIDE ./manage.py shell
"""
import os
import argparse
import pandas as pd
import requests
if __name__ != "__main__":
print(__name__)
raise Exception("Script has to be run directly")
# getting args from command line
parser = argparse.ArgumentParser(
description='Country and Region Insertion')
parser.add_argument('--address', '-a', dest='address',
action='store', default='http://127.0.0.1:8000/api',
help='Adresse of the web REST API')
parser.add_argument('--token', '-t', dest='token',
action='store', default='NO_TOKEN_PROVIDED',
help='Permanent token to use')
args = parser.parse_args()
api_address = str(args.address)
api_token = str(args.token)
tmp = os.path.join(os.path.realpath(__file__), '../../assets/country.csv')
country_file_loc = os.path.abspath(tmp)
tmp = os.path.join(os.path.realpath(__file__),
'../../assets/alpha-conv-table.csv')
conv_alpha_file_loc = os.path.abspath(tmp)
data = pd.read_csv(country_file_loc, sep=',', header=0,
dtype=object).fillna('')
data = data.drop(["Least Developed Countries (LDC)",
"Land Locked Developing Countries (LLDC)",
"Small Island Developing States (SIDS)",
"Developed / Developing Countries"],
axis=1)
# Need to load the information for converting
# Countries alpha-3 code to alpha-2 code
data_conv = pd.read_csv(conv_alpha_file_loc, sep=',',
header=0, na_filter=False)
conv_alpha_code = {}
for index, row in data_conv.iterrows():
conv_alpha_code[row["alpha-3"]] = row["alpha-2"]
def make_post(address, data):
h = {'Authorization': 'Token ' + api_token}
r = requests.post(address, headers=h, data=data)
if r.status_code is 403:
raise Exception("Authentification failed")
if not r.ok:
print(r.content)
print(data)
return r.ok
def save_country(attributes):
a = api_address + "/country/"
return make_post(a, attributes)
file_attr = ["Region Code", "Region Name",
"Sub-region Code", "Sub-region Name",
"Intermediate Region Code", "Intermediate Region Name",
"ISO-alpha3 Code", "Country or Area"]
db_attr = [
'region_un_code', 'region_name',
'sub_region_un_code', 'sub_region_name',
'intermediate_region_un_code', 'intermediate_region_name',
'iso_alpha3_code', 'name']
for index, row in data.iterrows():
country_attributes = {}
for f_a, db_a in zip(file_attr, db_attr):
country_attributes[db_a] = str(row[f_a])
tmp_c = str(row['ISO-alpha3 Code'])
if tmp_c in conv_alpha_code.keys():
country_attributes['iso_alpha2_code'] = conv_alpha_code[tmp_c]
save_country(country_attributes)
else:
print("ignoring country :", country_attributes)
#!/usr/bin/env python3
"""
Script to insert the country data in the database
IT HAS TO BE RUN INSIDE ./manage.py shell
"""
import argparse
import os
import pandas as pd
import requests
import json
if __name__ != "__main__":
print(__name__)
raise Exception("Script has to be run directly")
# getting args from command line
parser = argparse.ArgumentParser(
description='Country and Region Insertion')
parser.add_argument('--address', '-a', dest='address',
action='store', default='http://127.0.0.1:8000/api',
help='Adresse of the web REST API')
parser.add_argument('--token', '-t', dest='token',
action='store', default='NO_TOKEN_PROVIDED',
help='Permanent token to use')
args = parser.parse_args()
api_address = str(args.address)
api_token = str(args.token)
tmp = os.path.join(os.path.realpath(__file__),
'../../assets/destinations_extracted.csv')
destinations_path = os.path.abspath(tmp)
data = pd.read_csv(destinations_path, sep=',', header=0,
dtype=object)
def make_post(address, data):
h = {'Authorization': 'Token ' + api_token}
r = requests.post(address, headers=h, data=data)
if r.status_code is 403:
raise Exception("Authentification failed")
print(r.content)
return json.loads(r.content)["id"]
inserted_city_cc = {}
def insert_city(city_name, country):
if not city_name + country in inserted_city_cc.keys():
address = api_address + "/city/"
data = {'country': country, 'name': city_name}
inserted_city_cc[city_name + country] = make_post(address, data)
return inserted_city_cc[city_name + country]
def insert_university(name):
address = api_address + "/university/"
data = {'name': name}
return make_post(address, data)
def insert_main_campus(name, city_id, univ_id, lat, lon):
address = api_address + "/campus/"
data = {'name': "Campus " + name, 'city': city_id, 'lat': lat,
'lon': lon, 'is_main_campus': True,
'university': univ_id}
return make_post(address, data)
for index, row in data.iterrows():
univ, city, country, lat, lon = row
lat = round(float(lat), 6)
lon = round(float(lon), 6)
city_id = insert_city(city, country)
univ_id = insert_university(univ)
m_c_id = insert_main_campus(univ, city_id, univ_id, lat, lon)
#OutGoing_REX_dev
pandas # usefull for reading csv but necessary for the app to work
requests
Chargement des données initiales pour l'application
=====
Pour charger des données dans l'application, tout se passe via l'API du site.
Pour charger des données de base dans l'application, il suffit d'utiliser les classes `loadCountries`, `loadUniversities`, etc... disponibles dans `backend/load_data/loading_scripts`.
Autrement, pour tout charger d'un coup (et créer un administrateur avec pour identifiant "admin" et pour mot de passe "admin" **PENSER À LE CHANGER EN PRODUCTION...**), il suffit de passer par le shell de Django :
- Créer un superutilisateur :
```bash
./manage.py createsuperuser
./manage.py shell
```
- Créer un token dans l'administration (pour l'administrateur que vous venez de créer).
**CE TOKEN DOIT RESTER SECRET**.
- Lancer les scripts pour insérer les données :
```
./backend/utils/insert_country.py -t <Token> [-a <server adress>]
./backend/utils/insert_universities.py-t <Token> [-a <Adresse-de-l'API>]
puis
```python
from backend.load_data import load_all
load_all()
```
Ces scripts sont _unit-testés_ donc ils devraient fonctionner.
\ No newline at end of file
......@@ -17,4 +17,5 @@ pytest-django==3.4.2
pytest-cov==2.5.1
pytest-xdist==1.23.0
flake8
django-debug-toolbar==1.9.1
\ No newline at end of file
django-debug-toolbar==1.9.1
pandas
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment