Everything is ok

docs/load.md

 Chargement des données initiales pour l'application
 =====
 
-[SECTION QUI VA ÉVOLUER]
-
-Chargement des données des pays :
+Pour charger des données dans l'application, tout se passe via l'API du site.
 
+- Créer un superutilisateur : 
 ```bash
-./manage.py shell
+./manage.py createsuperuser
 ```
 
-Puis à l'intérieur de ce shell :
-```python
-%run ./rex/utils/insert_country.py
-```
+- Créer un token dans l'administration (pour l'administrateur que vous venez de créer).
+**CE TOKEN DOIT RESTER SECRET**.
 
-Pour les universitées, toujours à l'intérieur du shell `./manage.py shell` :
-```python
-%run ./rex/utils/insert_universities.py
-```
\ No newline at end of file
+- Lancer les scripts pour insérer les données :
+```
+./rex/utils/insert_country.py -t <Token> [-a <server adress>]
+./rex/utils/insert_universities.py-t <Token> [-a <Adresse-de-l'API>]
+```

rex/assets/country.csv

@@ -123,6 +123,7 @@
 "001","Monde","142","Asie","143","Asie centrale","","","Turkménistan","795","TKM","","x","","Developing"
 "001","Monde","142","Asie","143","Asie centrale","","","Ouzbékistan","860","UZB","","x","","Developing"
 "001","Monde","142","Asie","030","Asie orientale","","","Chine","156","CHN","","","","Developing"
+"001","Monde","142","Asie","030","Asie orientale","","","Taïwan","1589","TWN","","","","Developing"
 "001","Monde","142","Asie","030","Asie orientale","","","Chine, région administrative spéciale de Hong Kong","344","HKG","","","","Developing"
 "001","Monde","142","Asie","030","Asie orientale","","","Chine, région administrative spéciale de Macao","446","MAC","","","","Developing"
 "001","Monde","142","Asie","030","Asie orientale","","","République populaire démocratique de Corée","408","PRK","","","","Developing"

rex/assets/destinations_extracted.csv

-"University","lat","lon"
-"Technische Universitat Ilmenau",50.6828446,10.9383699
-"Universidad Del Salvador",-34.59941195,-58.392732040256
-"Swinburne University Of Technology",-37.85240845,144.99182255
-"Technische Universitat Graz",47.06817255,15.4497318287299
-"Universiteit Gent",51.0225378,3.71068134989542
-"Universidade Federal De Minas Gerais",-19.86868095,-43.962194217906
-"Universidade Federal De Campina Grande",-7.05925115,-37.2744434555441
-"Universidade Estadual De Campinas (Unicamp)",-22.8224781,-47.0642599309425
-"Universidade Caxias Do Sul",-29.16253545,-51.1483650151909
-"Pontificia Universidade Catolica Do Parana",-23.29907785,-51.2102693622486
-"Universidade Federal Do Parana",-23.5984464,-51.6732887712978
-"Universidade Tecnologica Federal Do Parana",-25.0521636,-50.131186381692
-"Universidade Federal De Santa Catarina",-27.5823901,-48.5050400209765
-"Universidade Federal De Itajuba",-22.4137971,-45.4502726
-"Universidade Federal Da Paraiba",-6.7505593,-35.6454502176197
-"Universidade Federal Juiz De Fora",-21.7768942,-43.3691810386558
-"Universidad Estadual De Maringa",-23.77903,-53.3252683935028
-"Universidade Federal De Pernambuco",-8.05254345,-34.9510286675945
-"Universidade Federal De Rio De Janeiro",-22.2459542,-42.5232866424185
-"Escola Politecnica Da Universidade De Sao Paulo",-23.5552106,-46.7310401128074
-"Universidade Federal De Uberlandia",-18.941683,-48.2148699674138
-"Technical University Of Sofia",42.6559773,23.35512261205
-"Universidad Arturo Prat",-33.43531325,-70.662435711441
-"Pontificia Universidad Catolica De Valparaiso",-33.0449276,-71.6067964701924
-"Universidad De Valparaiso",-32.7414062,-70.7024233560544
-"Universidad Tecnica Federico Santa Maria",-33.0354207,-71.5970503
-"Escuela Colombiana De Ingenieria",4.78305195,-74.0442040297156
-"Universidad Autonoma De Bucaramanga",7.1168832,-73.1051629
-"Universidad Eafit",6.10464025,-75.4557833087589
-"Pohang University Of Science And Technology",36.0172898,129.323920997982
-"Hanyang University",37.5551257,127.0447098
-"Kookmin University",37.61147915,126.996468815557
-"Escuela Politecnica Nacional De Quito",-0.2101692,-78.4886323
-"Universidad De Murcia",38.0147121,-1.1676358
-"Universidad De Zaragoza",41.6836311,-0.885944249252812
-"Oulu University Of Applied Sciences",65.00861805,25.4726477466852
-"Tampere University Of Technology",61.44964205,23.8587746189096
-"Universita Della Calabria",39.3614022,16.2263106
-"Universita Di Bologna",44.49964345,11.354422008333
-"Universita Degli Studi Di Salerno",40.75586365,14.7804889453213
-"Universita Degli Studi Di Genova",44.4148861,8.9266707
-"Politecnico Di Milano",45.4790288,9.23221040387672
-"Universita Degli Studi Di Napoli Federico Ii",40.84526205,14.2512487738249
-"Chiba Institute Of Technology",35.6888667,140.02117559587
-"Chiba University",35.62786925,140.103465749158
-"Tohoku University",38.2530945,140.8736593
-"Universite Libanaise",33.8724848,35.5194311623001
-"Université Saint-Esprit De Kaslik",33.98168505,35.6192920532997
-"Universidad Autonoma De Aguascalientes",21.8812194,-102.2924759
-"Universidad De Guanajuato",19.7242185,-101.2440472
-"Universidad Anahuac Mayab",21.11060445,-89.6123220044136
-"Instituto Politecnico Nacional",21.1358397,-86.8493286
-"Universidad Iberoamericana",25.445315,-101.0046875
-"Upaep",19.2818385,-98.4397356
-"Instituto Tecnologico De Zacatepec",18.65203335,-99.1833987516266
-"Norwegian University Of Science And Technology",63.4103583,10.4087418620805
-"University Twente",52.2380139,6.8566761
-"Agh - University Of Science And Technology",50.0657033,19.9189586670586
-"Politechnika Krakowska",50.0717009,19.941563
-"Universidade Da Beira Interior",40.2780936,-7.50834821105263
-"City University Of Hong Kong",22.34000115,114.169702912423
-"Technical University Of Kosice",48.73280395,21.244194264458
-"Chalmers University Of Technology",57.6896523,11.9766811023544
-"Lulea University Of Technology",65.61991265,22.1410701
-"Ecole Polytechnique Federale De Lausanne",46.5186594,6.566561505148
-"National Chiao Tung University",24.78676765,120.997244116807
-"National Taiwan University Of Science And Technology",25.01350785,121.541707560048
+"university","city","country","lat","lon"
+"Technische Universitat Ilmenau","Ilmenau","DE",50.68386845,10.9329051768032
+"Universidad Del Salvador","Buenos Aires","AR",-34.5562653,-58.730053914754
+"Swinburne University Of Technology","Victoria","AU",-37.85240845,144.99182255
+"Technische Universitat Graz","Graz","AT",47.05821,15.460195677136
+"Universiteit Gent","Gent","BE",51.0450413,3.7255573
+"Universidade Federal De Minas Gerais","Belo Horizonte","BR",-19.8718898,-43.9630925706377
+"Universidade Federal De Campina Grande","Campina Grande","BR",-6.76287395,-38.2281578339701
+"Universidade Estadual De Campinas (Unicamp)","Campinas-Sao Paulo","BR",-22.8224781,-47.0642599309425
+"Universidade Caxias Do Sul","Caxias Do Sul","BR",-29.16253545,-51.1483650151909
+"Pontificia Universidade Catolica Do Parana","Curitiba","BR",-23.29907785,-51.2102693622486
+"Universidade Federal Do Parana","Curitiba","BR",-23.5984464,-51.6732887712978
+"Universidade Tecnologica Federal Do Parana","Curitiba","BR",-25.0521636,-50.131186381692
+"Universidade Federal De Santa Catarina","Florianópolis","BR",-27.6033738,-48.5224633956674
+"Universidade Federal De Itajuba","Itajuba","BR",-22.4137971,-45.4502726
+"Universidade Federal Da Paraiba","João Pessoa","BR",-6.7505593,-35.6454502176197
+"Universidade Federal Juiz De Fora","Juiz De Fora","BR",-21.7768942,-43.3691810386558
+"Universidad Estadual De Maringa","Maringa","BR",-23.77903,-53.3252683935028
+"Universidade Federal De Pernambuco","Recife","BR",-8.05254345,-34.9510286675945
+"Universidade Federal De Rio De Janeiro","Rio De Janeiro","BR",-22.2459542,-42.5232866424185
+"Escola Politecnica Da Universidade De Sao Paulo","Sao Paulo","BR",-23.5552106,-46.7310401128074
+"Universidade Federal De Uberlandia","Uberlandia","BR",-18.941683,-48.2148699674138
+"Technical University Of Sofia","Sofia","BG",42.6559773,23.35512261205
+"Universidad Arturo Prat","Iquique","CL",-23.6455954,-70.394657443203
+"Pontificia Universidad Catolica De Valparaiso","Valparaiso","CL",-33.0449276,-71.6067964701924
+"Universidad De Valparaiso","Valparaiso","CL",-32.7414062,-70.7024233560544
+"Universidad Tecnica Federico Santa Maria","Valparaiso","CL",-33.03510935,-71.5952505079485
+"Escuela Colombiana De Ingenieria","Bogota","CO",4.78305195,-74.0442040297156
+"Universidad Autonoma De Bucaramanga","Bucaramanga","CO",7.1168832,-73.1051629
+"Universidad Eafit","Medellin","CO",6.10464025,-75.4557833087589
+"Pohang University Of Science And Technology","Pohang","KR",36.0172898,129.323920997982
+"Hanyang University","Seoul","KR",37.5558418,127.0438918
+"Kookmin University","Seoul","KR",37.61147915,126.996468815557
+"Seoul National University Of Science And Technology Seoultech","Seoul","KR",37.6324854,127.079487800846
+"Escuela Politecnica Nacional De Quito","Quito","EC",-0.21033215,-78.4900641291908
+"Universidad De Murcia","Murcia","ES",38.0147121,-1.1676358
+"Universidad De Zaragoza","Zaragoza","ES",41.6836311,-0.885944249252812
+"Oulu University Of Applied Sciences","Oulu","FI",65.00861805,25.4726477466852
+"Tampere University Of Technology","Tampere","FI",61.44964205,23.8587746189096
+"Universita Della Calabria","Arcavacata","IT",39.3614022,16.2263106
+"Universita Di Bologna","Bologna","IT",44.50134225,11.3591361693006
+"Universita Degli Studi Di Salerno","Fisciano","IT",40.75586365,14.7804889453213
+"Universita Degli Studi Di Genova","Genova","IT",44.4148861,8.9266707
+"Politecnico Di Milano","Milan","IT",45.4790288,9.23221040387672
+"Universita Degli Studi Di Napoli Federico Ii","Napoli","IT",40.84526205,14.2512487738249
+"Chiba Institute Of Technology","Chiba","JP",35.6888667,140.02117559587
+"Chiba University","Chiba","JP",35.62786925,140.103465749158
+"Tohoku University","Sendai","JP",38.2530945,140.8736593
+"Universite Libanaise","Beyrouth","LB",33.8724848,35.5194311623001
+"Université Saint-Esprit De Kaslik","Kaslik","LB",33.98265035,35.620356742624
+"Universidad Autonoma De Aguascalientes","Aguascalientes","MX",21.9142883,-102.32028877891
+"Universidad De Guanajuato","Guanajuato","MX",19.7242185,-101.2440472
+"Universidad Anahuac Mayab","Merida Yucatan","MX",21.11060445,-89.6123220044136
+"Instituto Politecnico Nacional","Mexico","MX",19.4724472,-99.1373193
+"Universidad Iberoamericana","Mexico","MX",25.445818,-101.0056267
+"Upaep","Pueblar, Pue","MX",19.2818385,-98.4397356
+"Instituto Tecnologico De Zacatepec","Zacatepec De Hidalgo","MX",18.65203335,-99.1833987516266
+"Norwegian University Of Science And Technology","Trondheim","NO",63.4103583,10.4087418620805
+"University Twente","Enschede","NL",52.2233862,6.88542315997116
+"Agh - University Of Science And Technology","Cracovie","PL",50.0657033,19.9189586670586
+"Politechnika Krakowska","Cracovie","PL",50.0754784,19.9089536
+"Universidade Da Beira Interior","Covilhã","PT",40.2780936,-7.50834821105263
+"City University Of Hong Kong","Hong Kong","HK",22.34000115,114.169702912423
+"Technical University Of Kosice","Kosice","SK",48.73280395,21.244194264458
+"Chalmers University Of Technology","Goteborg","SE",57.6896523,11.9766811023544
+"Lulea University Of Technology","Lulea","SE",65.6170445,22.1370606335398
+"Ecole Polytechnique Federale De Lausanne","Lausanne","CH",46.5186594,6.566561505148
+"National Chiao Tung University","Hsinchu","TW",24.78676765,120.997244116807
+"National Taiwan University Of Science And Technology","Taipei","TW",25.01350785,121.541707560048

rex/migrations/0001_initial.py

-# Generated by Django 2.0.3 on 2018-04-25 13:44
+# Generated by Django 2.0.3 on 2018-04-25 14:48
 
 import django.core.validators
 from django.db import migrations, models
@@ -57,7 +57,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=200)),
-                ('acronym', models.CharField(max_length=20)),
+                ('acronym', models.CharField(blank=True, max_length=20, null=True)),
                 ('logo', models.ImageField(default='path/to/my/default/image.jpg', max_length=250, upload_to=rex.utils.friendly_path.friendly_path('uploads/universities/logos/', 'name'))),
                 ('main_campus', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='main_campus_of', to='rex.MainCampus')),
             ],

rex/models/university/__init__.py

-from .university import University  # noqa: F401
+from .university import University, UniversityViewSet  # noqa: F401
 from .campus import MainCampus, MainCampusViewSet  # noqa: F401

rex/models/university/campus.py

@@ -38,13 +38,9 @@ class MainCampus(Campus):
 
 
 class MainCampusSerializer(serializers.ModelSerializer):
-    gps_coordinates = serializers.SerializerMethodField()
     modified_by = serializers.SerializerMethodField()
     modified_date = serializers.SerializerMethodField()
 
-    def get_gps_coordinates(self, obj):
-        return obj.location()
-
     def get_modified_by(self, obj):
         return obj.modified_by()
 
@@ -52,8 +48,8 @@ class MainCampusSerializer(serializers.ModelSerializer):
         return obj.modified_date()
 
     class Meta:
-        model = City
-        fields = ('gps_coordinates', 'modified_by', 'modified_date', 'id')
+        model = MainCampus
+        fields = ('lat', 'lon', 'name', 'city', 'modified_by', 'modified_date', 'id')
 
 
 class MainCampusViewSet(viewsets.ModelViewSet):  # TODO switch to dict

rex/models/university/university.py

 from django.db import models
+from rest_framework import serializers, viewsets, permissions
 
 from rex.utils.friendly_path import friendly_path
 from rex.models.university.campus import MainCampus
@@ -11,7 +12,7 @@ class University(models.Model):
     Model storing information about universities
     """
     name = models.CharField(max_length=200)
-    acronym = models.CharField(max_length=20)
+    acronym = models.CharField(max_length=20, null=True, blank=True)
     logo = models.ImageField(upload_to=path_and_rename, max_length=250,
                              default='path/to/my/default/image.jpg')  # TODO CHANGE !
 
@@ -20,3 +21,15 @@ class University(models.Model):
 
     def location(self):
         return self.main_campus.location()  # pylint: disable=E1101
+
+
+class UniversitySerializer(serializers.ModelSerializer):
+    class Meta:
+        model = University
+        fields = ('name', 'acronym', 'logo', 'main_campus', 'id')
+
+
+class UniversityViewSet(viewsets.ModelViewSet):  # TODO switch to dict
+    permission_classes = (permissions.DjangoModelPermissions,)
+    queryset = University.objects.all()  # pylint: disable=E1101
+    serializer_class = UniversitySerializer

rex/templates/rex/home.html

@@ -54,7 +54,7 @@
         var universities_markers = L.markerClusterGroup();
 
         {% for univ in universities %}
-            L.marker([{{ univ.location_lat }}, {{ univ.location_lon }}]).addTo(universities_markers)
+            L.marker([{{ univ.loc.lat }}, {{ univ.loc.lon }}]).addTo(universities_markers)
             .bindPopup("<b>{{ univ.name }}</b>");
         {% endfor %}
 
@@ -77,8 +77,8 @@
     
         mymap.on('click', onMapClick);
     </script>
-    {% for univ in universities %}
+    <!-- {% for univ in universities %}
         <p>{{ univ.country.name }}</p>
-    {% endfor %}
+    {% endfor %} -->
 {% endblock %}
     
\ No newline at end of file

rex/urls.py

@@ -2,7 +2,7 @@ from django.conf.urls import url, include
 from rex import views
 from rest_framework import routers
 from rex.models.location import CountryViewSet, RegionViewSet, CityViewSet
-from rex.models.university import MainCampusViewSet
+from rex.models.university import MainCampusViewSet, UniversityViewSet
 from rest_framework.documentation import include_docs_urls
 
 
@@ -17,5 +17,6 @@ router.register(r'country', CountryViewSet)
 router.register(r'region', RegionViewSet)
 router.register(r'city', CityViewSet)
 router.register(r'main-campus', MainCampusViewSet)
+router.register(r'university', UniversityViewSet)
 
 urlpatterns += [url(r'^api/', include(router.urls))]

rex/utils/extract_universities.py

@@ -9,6 +9,7 @@ import csv
 import os
 import time
 from geopy.geocoders import Nominatim
+import reverse_geocoder as rg
 
 tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations.csv')
 destinations_path = os.path.abspath(tmp)
@@ -33,7 +34,7 @@ with open(destinations_path, 'rt') as input:
         for row in reader:
             # handle the header
             if i == 0:
-                header = ['University', 'lat', 'lon']
+                header = ['university', 'city', 'country', 'lat', 'lon']
                 spamwriter.writerow(header)
                 i += 1
             else:
@@ -45,12 +46,13 @@ with open(destinations_path, 'rt') as input:
                         break
                     except:  # noqa: E722
                         print("error during query, retrying")
-                        time.sleep(1)
-
-                print(location)
+                        time.sleep(0.5)
                 if location is not None:
-                    spamwriter.writerow(
-                        [row[2], location.latitude, location.longitude])
+                    coord = (location.latitude, location.longitude)
+                    res = rg.search(coord, verbose=False)
+                    line = [row[2], row[1], res[0]['cc'], location.latitude, location.longitude]
+                    print(line)
+                    spamwriter.writerow(line)
                 else:
                     failed.append(query)
 

rex/utils/insert_universities.py

@@ -5,24 +5,78 @@ Script to insert the country data in the database
 
 IT HAS TO BE RUN INSIDE ./manage.py shell
 """
-import csv
+import argparse
 import os
+import pandas as pd
+import requests
+import json
+
+if __name__ != "__main__":
+    print(__name__)
+    raise Exception("Script has to be run directly")
+
+# getting args from command line
+parser = argparse.ArgumentParser(
+    description='Country and Region Insertion')
+
+parser.add_argument('--address', '-a', dest='address',
+                    action='store', default='http://127.0.0.1:8000/api',
+                    help='Adresse of the web REST API')
+
+parser.add_argument('--token', '-t', dest='token',
+                    action='store', default='NO_TOKEN_PROVIDED',
+                    help='Permanent token to use')
+
+args = parser.parse_args()
+
+api_address = str(args.address)
+api_token = str(args.token)
 
-from rex.models.university import University
 
 tmp = os.path.join(os.path.realpath(__file__),
                    '../../assets/destinations_extracted.csv')
 destinations_path = os.path.abspath(tmp)
 
-if not os.path.isfile(destinations_path):
-    print(destinations_path)
-    raise Exception("Missing file containing country data")
+data = pd.read_csv(destinations_path, sep=',', header=0,
+                   dtype=object)
+
+
+def make_post(address, data):
+    h = {'Authorization': 'Token ' + api_token}
+    r = requests.post(address, headers=h, data=data)
+    if r.status_code is 403:
+        raise Exception("Authentification failed")
+    print(r.content)
+    return json.loads(r.content)["id"]
+
+
+inserted_city_cc = {}
+
+
+def insert_city(city_name, country):
+    if not city_name + country in inserted_city_cc.keys():
+        address = api_address + "/city/"
+        data = {'country': country, 'name': city_name}
+        inserted_city_cc[city_name + country] = make_post(address, data)
+    return inserted_city_cc[city_name + country]
+
+
+def insert_main_campus(name, city_id, lat, lon):
+    address = api_address + "/main-campus/"
+    data = {'name': name, 'city': city_id, 'lat': lat, 'lon': lon}
+    return make_post(address, data)
+
+
+def insert_university(name, main_campus_id):
+    address = api_address + "/university/"
+    data = {'name': name, 'main_campus': main_campus_id}
+    return make_post(address, data)
 
-with open(destinations_path, 'rt') as f:
-    reader = csv.reader(f)
 
-    for row in reader:
-        if row[0] != 'University':
-            u = University(name=row[0], acronym='NULL', localization_lat=float(
-                row[1]), localization_lon=float(row[2]))
-            u.save()
+for index, row in data.iterrows():
+    univ, city, country, lat, lon = row
+    lat = round(float(lat), 6)
+    lon = round(float(lon), 6)
+    city_id = insert_city(city, country)
+    m_c_id = insert_main_campus(univ, city_id, lat, lon)
+    insert_university(univ, m_c_id)

rex/views.py

@@ -4,4 +4,6 @@ from rex.models.university import University
 
 def home(request):
     universities = University.objects.all()
+    for univ in universities:
+        univ.loc = univ.location()
     return render(request, 'rex/home.html', {'universities': universities})