From a1ea001556f20a74b6cb1df14536d24511af2182 Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Wed, 25 Apr 2018 17:01:38 +0200 Subject: [PATCH] Everything is ok --- docs/load.md | 22 ++--- rex/assets/country.csv | 1 + rex/assets/destinations_extracted.csv | 137 +++++++++++++------------- rex/migrations/0001_initial.py | 4 +- rex/models/university/__init__.py | 2 +- rex/models/university/campus.py | 8 +- rex/models/university/university.py | 15 ++- rex/templates/rex/home.html | 6 +- rex/urls.py | 3 +- rex/utils/extract_universities.py | 14 +-- rex/utils/insert_universities.py | 78 ++++++++++++--- rex/views.py | 2 + 12 files changed, 180 insertions(+), 112 deletions(-) mode change 100644 => 100755 rex/utils/extract_universities.py diff --git a/docs/load.md b/docs/load.md index 0df3133b..c70c0930 100644 --- a/docs/load.md +++ b/docs/load.md @@ -1,20 +1,18 @@ Chargement des données initiales pour l'application ===== -[SECTION QUI VA ÉVOLUER] - -Chargement des données des pays : +Pour charger des données dans l'application, tout se passe via l'API du site. +- Créer un superutilisateur : ```bash -./manage.py shell +./manage.py createsuperuser ``` -Puis à l'intérieur de ce shell : -```python -%run ./rex/utils/insert_country.py -``` +- Créer un token dans l'administration (pour l'administrateur que vous venez de créer). +**CE TOKEN DOIT RESTER SECRET**. -Pour les universitées, toujours à l'intérieur du shell `./manage.py shell` : -```python -%run ./rex/utils/insert_universities.py -``` \ No newline at end of file +- Lancer les scripts pour insérer les données : +``` +./rex/utils/insert_country.py -t [-a ] +./rex/utils/insert_universities.py-t [-a ] +``` diff --git a/rex/assets/country.csv b/rex/assets/country.csv index 521212a7..72b8117a 100644 --- a/rex/assets/country.csv +++ b/rex/assets/country.csv @@ -123,6 +123,7 @@ "001","Monde","142","Asie","143","Asie centrale","","","Turkménistan","795","TKM","","x","","Developing" "001","Monde","142","Asie","143","Asie centrale","","","Ouzbékistan","860","UZB","","x","","Developing" "001","Monde","142","Asie","030","Asie orientale","","","Chine","156","CHN","","","","Developing" +"001","Monde","142","Asie","030","Asie orientale","","","Taïwan","1589","TWN","","","","Developing" "001","Monde","142","Asie","030","Asie orientale","","","Chine, région administrative spéciale de Hong Kong","344","HKG","","","","Developing" "001","Monde","142","Asie","030","Asie orientale","","","Chine, région administrative spéciale de Macao","446","MAC","","","","Developing" "001","Monde","142","Asie","030","Asie orientale","","","République populaire démocratique de Corée","408","PRK","","","","Developing" diff --git a/rex/assets/destinations_extracted.csv b/rex/assets/destinations_extracted.csv index fdeb2027..33c9878e 100644 --- a/rex/assets/destinations_extracted.csv +++ b/rex/assets/destinations_extracted.csv @@ -1,68 +1,69 @@ -"University","lat","lon" -"Technische Universitat Ilmenau",50.6828446,10.9383699 -"Universidad Del Salvador",-34.59941195,-58.392732040256 -"Swinburne University Of Technology",-37.85240845,144.99182255 -"Technische Universitat Graz",47.06817255,15.4497318287299 -"Universiteit Gent",51.0225378,3.71068134989542 -"Universidade Federal De Minas Gerais",-19.86868095,-43.962194217906 -"Universidade Federal De Campina Grande",-7.05925115,-37.2744434555441 -"Universidade Estadual De Campinas (Unicamp)",-22.8224781,-47.0642599309425 -"Universidade Caxias Do Sul",-29.16253545,-51.1483650151909 -"Pontificia Universidade Catolica Do Parana",-23.29907785,-51.2102693622486 -"Universidade Federal Do Parana",-23.5984464,-51.6732887712978 -"Universidade Tecnologica Federal Do Parana",-25.0521636,-50.131186381692 -"Universidade Federal De Santa Catarina",-27.5823901,-48.5050400209765 -"Universidade Federal De Itajuba",-22.4137971,-45.4502726 -"Universidade Federal Da Paraiba",-6.7505593,-35.6454502176197 -"Universidade Federal Juiz De Fora",-21.7768942,-43.3691810386558 -"Universidad Estadual De Maringa",-23.77903,-53.3252683935028 -"Universidade Federal De Pernambuco",-8.05254345,-34.9510286675945 -"Universidade Federal De Rio De Janeiro",-22.2459542,-42.5232866424185 -"Escola Politecnica Da Universidade De Sao Paulo",-23.5552106,-46.7310401128074 -"Universidade Federal De Uberlandia",-18.941683,-48.2148699674138 -"Technical University Of Sofia",42.6559773,23.35512261205 -"Universidad Arturo Prat",-33.43531325,-70.662435711441 -"Pontificia Universidad Catolica De Valparaiso",-33.0449276,-71.6067964701924 -"Universidad De Valparaiso",-32.7414062,-70.7024233560544 -"Universidad Tecnica Federico Santa Maria",-33.0354207,-71.5970503 -"Escuela Colombiana De Ingenieria",4.78305195,-74.0442040297156 -"Universidad Autonoma De Bucaramanga",7.1168832,-73.1051629 -"Universidad Eafit",6.10464025,-75.4557833087589 -"Pohang University Of Science And Technology",36.0172898,129.323920997982 -"Hanyang University",37.5551257,127.0447098 -"Kookmin University",37.61147915,126.996468815557 -"Escuela Politecnica Nacional De Quito",-0.2101692,-78.4886323 -"Universidad De Murcia",38.0147121,-1.1676358 -"Universidad De Zaragoza",41.6836311,-0.885944249252812 -"Oulu University Of Applied Sciences",65.00861805,25.4726477466852 -"Tampere University Of Technology",61.44964205,23.8587746189096 -"Universita Della Calabria",39.3614022,16.2263106 -"Universita Di Bologna",44.49964345,11.354422008333 -"Universita Degli Studi Di Salerno",40.75586365,14.7804889453213 -"Universita Degli Studi Di Genova",44.4148861,8.9266707 -"Politecnico Di Milano",45.4790288,9.23221040387672 -"Universita Degli Studi Di Napoli Federico Ii",40.84526205,14.2512487738249 -"Chiba Institute Of Technology",35.6888667,140.02117559587 -"Chiba University",35.62786925,140.103465749158 -"Tohoku University",38.2530945,140.8736593 -"Universite Libanaise",33.8724848,35.5194311623001 -"Université Saint-Esprit De Kaslik",33.98168505,35.6192920532997 -"Universidad Autonoma De Aguascalientes",21.8812194,-102.2924759 -"Universidad De Guanajuato",19.7242185,-101.2440472 -"Universidad Anahuac Mayab",21.11060445,-89.6123220044136 -"Instituto Politecnico Nacional",21.1358397,-86.8493286 -"Universidad Iberoamericana",25.445315,-101.0046875 -"Upaep",19.2818385,-98.4397356 -"Instituto Tecnologico De Zacatepec",18.65203335,-99.1833987516266 -"Norwegian University Of Science And Technology",63.4103583,10.4087418620805 -"University Twente",52.2380139,6.8566761 -"Agh - University Of Science And Technology",50.0657033,19.9189586670586 -"Politechnika Krakowska",50.0717009,19.941563 -"Universidade Da Beira Interior",40.2780936,-7.50834821105263 -"City University Of Hong Kong",22.34000115,114.169702912423 -"Technical University Of Kosice",48.73280395,21.244194264458 -"Chalmers University Of Technology",57.6896523,11.9766811023544 -"Lulea University Of Technology",65.61991265,22.1410701 -"Ecole Polytechnique Federale De Lausanne",46.5186594,6.566561505148 -"National Chiao Tung University",24.78676765,120.997244116807 -"National Taiwan University Of Science And Technology",25.01350785,121.541707560048 +"university","city","country","lat","lon" +"Technische Universitat Ilmenau","Ilmenau","DE",50.68386845,10.9329051768032 +"Universidad Del Salvador","Buenos Aires","AR",-34.5562653,-58.730053914754 +"Swinburne University Of Technology","Victoria","AU",-37.85240845,144.99182255 +"Technische Universitat Graz","Graz","AT",47.05821,15.460195677136 +"Universiteit Gent","Gent","BE",51.0450413,3.7255573 +"Universidade Federal De Minas Gerais","Belo Horizonte","BR",-19.8718898,-43.9630925706377 +"Universidade Federal De Campina Grande","Campina Grande","BR",-6.76287395,-38.2281578339701 +"Universidade Estadual De Campinas (Unicamp)","Campinas-Sao Paulo","BR",-22.8224781,-47.0642599309425 +"Universidade Caxias Do Sul","Caxias Do Sul","BR",-29.16253545,-51.1483650151909 +"Pontificia Universidade Catolica Do Parana","Curitiba","BR",-23.29907785,-51.2102693622486 +"Universidade Federal Do Parana","Curitiba","BR",-23.5984464,-51.6732887712978 +"Universidade Tecnologica Federal Do Parana","Curitiba","BR",-25.0521636,-50.131186381692 +"Universidade Federal De Santa Catarina","Florianópolis","BR",-27.6033738,-48.5224633956674 +"Universidade Federal De Itajuba","Itajuba","BR",-22.4137971,-45.4502726 +"Universidade Federal Da Paraiba","João Pessoa","BR",-6.7505593,-35.6454502176197 +"Universidade Federal Juiz De Fora","Juiz De Fora","BR",-21.7768942,-43.3691810386558 +"Universidad Estadual De Maringa","Maringa","BR",-23.77903,-53.3252683935028 +"Universidade Federal De Pernambuco","Recife","BR",-8.05254345,-34.9510286675945 +"Universidade Federal De Rio De Janeiro","Rio De Janeiro","BR",-22.2459542,-42.5232866424185 +"Escola Politecnica Da Universidade De Sao Paulo","Sao Paulo","BR",-23.5552106,-46.7310401128074 +"Universidade Federal De Uberlandia","Uberlandia","BR",-18.941683,-48.2148699674138 +"Technical University Of Sofia","Sofia","BG",42.6559773,23.35512261205 +"Universidad Arturo Prat","Iquique","CL",-23.6455954,-70.394657443203 +"Pontificia Universidad Catolica De Valparaiso","Valparaiso","CL",-33.0449276,-71.6067964701924 +"Universidad De Valparaiso","Valparaiso","CL",-32.7414062,-70.7024233560544 +"Universidad Tecnica Federico Santa Maria","Valparaiso","CL",-33.03510935,-71.5952505079485 +"Escuela Colombiana De Ingenieria","Bogota","CO",4.78305195,-74.0442040297156 +"Universidad Autonoma De Bucaramanga","Bucaramanga","CO",7.1168832,-73.1051629 +"Universidad Eafit","Medellin","CO",6.10464025,-75.4557833087589 +"Pohang University Of Science And Technology","Pohang","KR",36.0172898,129.323920997982 +"Hanyang University","Seoul","KR",37.5558418,127.0438918 +"Kookmin University","Seoul","KR",37.61147915,126.996468815557 +"Seoul National University Of Science And Technology Seoultech","Seoul","KR",37.6324854,127.079487800846 +"Escuela Politecnica Nacional De Quito","Quito","EC",-0.21033215,-78.4900641291908 +"Universidad De Murcia","Murcia","ES",38.0147121,-1.1676358 +"Universidad De Zaragoza","Zaragoza","ES",41.6836311,-0.885944249252812 +"Oulu University Of Applied Sciences","Oulu","FI",65.00861805,25.4726477466852 +"Tampere University Of Technology","Tampere","FI",61.44964205,23.8587746189096 +"Universita Della Calabria","Arcavacata","IT",39.3614022,16.2263106 +"Universita Di Bologna","Bologna","IT",44.50134225,11.3591361693006 +"Universita Degli Studi Di Salerno","Fisciano","IT",40.75586365,14.7804889453213 +"Universita Degli Studi Di Genova","Genova","IT",44.4148861,8.9266707 +"Politecnico Di Milano","Milan","IT",45.4790288,9.23221040387672 +"Universita Degli Studi Di Napoli Federico Ii","Napoli","IT",40.84526205,14.2512487738249 +"Chiba Institute Of Technology","Chiba","JP",35.6888667,140.02117559587 +"Chiba University","Chiba","JP",35.62786925,140.103465749158 +"Tohoku University","Sendai","JP",38.2530945,140.8736593 +"Universite Libanaise","Beyrouth","LB",33.8724848,35.5194311623001 +"Université Saint-Esprit De Kaslik","Kaslik","LB",33.98265035,35.620356742624 +"Universidad Autonoma De Aguascalientes","Aguascalientes","MX",21.9142883,-102.32028877891 +"Universidad De Guanajuato","Guanajuato","MX",19.7242185,-101.2440472 +"Universidad Anahuac Mayab","Merida Yucatan","MX",21.11060445,-89.6123220044136 +"Instituto Politecnico Nacional","Mexico","MX",19.4724472,-99.1373193 +"Universidad Iberoamericana","Mexico","MX",25.445818,-101.0056267 +"Upaep","Pueblar, Pue","MX",19.2818385,-98.4397356 +"Instituto Tecnologico De Zacatepec","Zacatepec De Hidalgo","MX",18.65203335,-99.1833987516266 +"Norwegian University Of Science And Technology","Trondheim","NO",63.4103583,10.4087418620805 +"University Twente","Enschede","NL",52.2233862,6.88542315997116 +"Agh - University Of Science And Technology","Cracovie","PL",50.0657033,19.9189586670586 +"Politechnika Krakowska","Cracovie","PL",50.0754784,19.9089536 +"Universidade Da Beira Interior","Covilhã","PT",40.2780936,-7.50834821105263 +"City University Of Hong Kong","Hong Kong","HK",22.34000115,114.169702912423 +"Technical University Of Kosice","Kosice","SK",48.73280395,21.244194264458 +"Chalmers University Of Technology","Goteborg","SE",57.6896523,11.9766811023544 +"Lulea University Of Technology","Lulea","SE",65.6170445,22.1370606335398 +"Ecole Polytechnique Federale De Lausanne","Lausanne","CH",46.5186594,6.566561505148 +"National Chiao Tung University","Hsinchu","TW",24.78676765,120.997244116807 +"National Taiwan University Of Science And Technology","Taipei","TW",25.01350785,121.541707560048 diff --git a/rex/migrations/0001_initial.py b/rex/migrations/0001_initial.py index 74b6aacc..fd5e485e 100644 --- a/rex/migrations/0001_initial.py +++ b/rex/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.0.3 on 2018-04-25 13:44 +# Generated by Django 2.0.3 on 2018-04-25 14:48 import django.core.validators from django.db import migrations, models @@ -57,7 +57,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=200)), - ('acronym', models.CharField(max_length=20)), + ('acronym', models.CharField(blank=True, max_length=20, null=True)), ('logo', models.ImageField(default='path/to/my/default/image.jpg', max_length=250, upload_to=rex.utils.friendly_path.friendly_path('uploads/universities/logos/', 'name'))), ('main_campus', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='main_campus_of', to='rex.MainCampus')), ], diff --git a/rex/models/university/__init__.py b/rex/models/university/__init__.py index 4880f312..fe8eac16 100644 --- a/rex/models/university/__init__.py +++ b/rex/models/university/__init__.py @@ -1,2 +1,2 @@ -from .university import University # noqa: F401 +from .university import University, UniversityViewSet # noqa: F401 from .campus import MainCampus, MainCampusViewSet # noqa: F401 diff --git a/rex/models/university/campus.py b/rex/models/university/campus.py index 4fb9c616..ec4bcb9a 100644 --- a/rex/models/university/campus.py +++ b/rex/models/university/campus.py @@ -38,13 +38,9 @@ class MainCampus(Campus): class MainCampusSerializer(serializers.ModelSerializer): - gps_coordinates = serializers.SerializerMethodField() modified_by = serializers.SerializerMethodField() modified_date = serializers.SerializerMethodField() - def get_gps_coordinates(self, obj): - return obj.location() - def get_modified_by(self, obj): return obj.modified_by() @@ -52,8 +48,8 @@ class MainCampusSerializer(serializers.ModelSerializer): return obj.modified_date() class Meta: - model = City - fields = ('gps_coordinates', 'modified_by', 'modified_date', 'id') + model = MainCampus + fields = ('lat', 'lon', 'name', 'city', 'modified_by', 'modified_date', 'id') class MainCampusViewSet(viewsets.ModelViewSet): # TODO switch to dict diff --git a/rex/models/university/university.py b/rex/models/university/university.py index a616dbe1..1964f29e 100644 --- a/rex/models/university/university.py +++ b/rex/models/university/university.py @@ -1,4 +1,5 @@ from django.db import models +from rest_framework import serializers, viewsets, permissions from rex.utils.friendly_path import friendly_path from rex.models.university.campus import MainCampus @@ -11,7 +12,7 @@ class University(models.Model): Model storing information about universities """ name = models.CharField(max_length=200) - acronym = models.CharField(max_length=20) + acronym = models.CharField(max_length=20, null=True, blank=True) logo = models.ImageField(upload_to=path_and_rename, max_length=250, default='path/to/my/default/image.jpg') # TODO CHANGE ! @@ -20,3 +21,15 @@ class University(models.Model): def location(self): return self.main_campus.location() # pylint: disable=E1101 + + +class UniversitySerializer(serializers.ModelSerializer): + class Meta: + model = University + fields = ('name', 'acronym', 'logo', 'main_campus', 'id') + + +class UniversityViewSet(viewsets.ModelViewSet): # TODO switch to dict + permission_classes = (permissions.DjangoModelPermissions,) + queryset = University.objects.all() # pylint: disable=E1101 + serializer_class = UniversitySerializer diff --git a/rex/templates/rex/home.html b/rex/templates/rex/home.html index 9e17223a..9fbdea31 100644 --- a/rex/templates/rex/home.html +++ b/rex/templates/rex/home.html @@ -54,7 +54,7 @@ var universities_markers = L.markerClusterGroup(); {% for univ in universities %} - L.marker([{{ univ.location_lat }}, {{ univ.location_lon }}]).addTo(universities_markers) + L.marker([{{ univ.loc.lat }}, {{ univ.loc.lon }}]).addTo(universities_markers) .bindPopup("{{ univ.name }}"); {% endfor %} @@ -77,8 +77,8 @@ mymap.on('click', onMapClick); - {% for univ in universities %} + {% endblock %} \ No newline at end of file diff --git a/rex/urls.py b/rex/urls.py index 16964cd8..f9239256 100644 --- a/rex/urls.py +++ b/rex/urls.py @@ -2,7 +2,7 @@ from django.conf.urls import url, include from rex import views from rest_framework import routers from rex.models.location import CountryViewSet, RegionViewSet, CityViewSet -from rex.models.university import MainCampusViewSet +from rex.models.university import MainCampusViewSet, UniversityViewSet from rest_framework.documentation import include_docs_urls @@ -17,5 +17,6 @@ router.register(r'country', CountryViewSet) router.register(r'region', RegionViewSet) router.register(r'city', CityViewSet) router.register(r'main-campus', MainCampusViewSet) +router.register(r'university', UniversityViewSet) urlpatterns += [url(r'^api/', include(router.urls))] diff --git a/rex/utils/extract_universities.py b/rex/utils/extract_universities.py old mode 100644 new mode 100755 index 1dbf6847..1f5bac3c --- a/rex/utils/extract_universities.py +++ b/rex/utils/extract_universities.py @@ -9,6 +9,7 @@ import csv import os import time from geopy.geocoders import Nominatim +import reverse_geocoder as rg tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations.csv') destinations_path = os.path.abspath(tmp) @@ -33,7 +34,7 @@ with open(destinations_path, 'rt') as input: for row in reader: # handle the header if i == 0: - header = ['University', 'lat', 'lon'] + header = ['university', 'city', 'country', 'lat', 'lon'] spamwriter.writerow(header) i += 1 else: @@ -45,12 +46,13 @@ with open(destinations_path, 'rt') as input: break except: # noqa: E722 print("error during query, retrying") - time.sleep(1) - - print(location) + time.sleep(0.5) if location is not None: - spamwriter.writerow( - [row[2], location.latitude, location.longitude]) + coord = (location.latitude, location.longitude) + res = rg.search(coord, verbose=False) + line = [row[2], row[1], res[0]['cc'], location.latitude, location.longitude] + print(line) + spamwriter.writerow(line) else: failed.append(query) diff --git a/rex/utils/insert_universities.py b/rex/utils/insert_universities.py index a77a3c5d..9119ac65 100755 --- a/rex/utils/insert_universities.py +++ b/rex/utils/insert_universities.py @@ -5,24 +5,78 @@ Script to insert the country data in the database IT HAS TO BE RUN INSIDE ./manage.py shell """ -import csv +import argparse import os +import pandas as pd +import requests +import json + +if __name__ != "__main__": + print(__name__) + raise Exception("Script has to be run directly") + +# getting args from command line +parser = argparse.ArgumentParser( + description='Country and Region Insertion') + +parser.add_argument('--address', '-a', dest='address', + action='store', default='http://127.0.0.1:8000/api', + help='Adresse of the web REST API') + +parser.add_argument('--token', '-t', dest='token', + action='store', default='NO_TOKEN_PROVIDED', + help='Permanent token to use') + +args = parser.parse_args() + +api_address = str(args.address) +api_token = str(args.token) -from rex.models.university import University tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations_extracted.csv') destinations_path = os.path.abspath(tmp) -if not os.path.isfile(destinations_path): - print(destinations_path) - raise Exception("Missing file containing country data") +data = pd.read_csv(destinations_path, sep=',', header=0, + dtype=object) + + +def make_post(address, data): + h = {'Authorization': 'Token ' + api_token} + r = requests.post(address, headers=h, data=data) + if r.status_code is 403: + raise Exception("Authentification failed") + print(r.content) + return json.loads(r.content)["id"] + + +inserted_city_cc = {} + + +def insert_city(city_name, country): + if not city_name + country in inserted_city_cc.keys(): + address = api_address + "/city/" + data = {'country': country, 'name': city_name} + inserted_city_cc[city_name + country] = make_post(address, data) + return inserted_city_cc[city_name + country] + + +def insert_main_campus(name, city_id, lat, lon): + address = api_address + "/main-campus/" + data = {'name': name, 'city': city_id, 'lat': lat, 'lon': lon} + return make_post(address, data) + + +def insert_university(name, main_campus_id): + address = api_address + "/university/" + data = {'name': name, 'main_campus': main_campus_id} + return make_post(address, data) -with open(destinations_path, 'rt') as f: - reader = csv.reader(f) - for row in reader: - if row[0] != 'University': - u = University(name=row[0], acronym='NULL', localization_lat=float( - row[1]), localization_lon=float(row[2])) - u.save() +for index, row in data.iterrows(): + univ, city, country, lat, lon = row + lat = round(float(lat), 6) + lon = round(float(lon), 6) + city_id = insert_city(city, country) + m_c_id = insert_main_campus(univ, city_id, lat, lon) + insert_university(univ, m_c_id) diff --git a/rex/views.py b/rex/views.py index 9c179c82..4879d729 100644 --- a/rex/views.py +++ b/rex/views.py @@ -4,4 +4,6 @@ from rex.models.university import University def home(request): universities = University.objects.all() + for univ in universities: + univ.loc = univ.location() return render(request, 'rex/home.html', {'universities': universities}) -- GitLab