populate universities added

README.md

@@ -18,3 +18,7 @@ puis à l'intérieur de ce shell
 ```
 %run ./rex/utils/insert_country.py
 ```
+To populate universities, also run in `./manage.py shell`:
+```
+%run ./rex/utils/insert_universities.py
+```
\ No newline at end of file

rex/assets/destinations.csv

+"Pays","Ville","Nom de l'établissement"
+"Allemagne","Hamburg","Technische Universitat Hamburg"
+"Allemagne","Ilmenau","Technische Universitat Ilmenau"
+"Argentine","Buenos Aires","Universidad Del Salvador"
+"Australie","Victoria","Swinburne University Of Technology"
+"Autriche","Graz","Technische Universitat Graz"
+"Belgique","Gent","Universiteit Gent"
+"Brésil","Belo Horizonte","Universidade Federal De Minas Gerais"
+"Brésil","Campina Grande","Universidade Federal De Campina Grande"
+"Brésil","Campinas-Sao Paulo","Universidade Estadual De Campinas (Unicamp)"
+"Brésil","Caxias Do Sul","Universidade Caxias Do Sul"
+"Brésil","Curitiba","Pontificia Universidade Catolica Do Parana"
+"Brésil","Curitiba","Universidade Federal Do Parana"
+"Brésil","Curitiba","Universidade Tecnologica Federal Do Parana"
+"Brésil","Florianópolis","Universidade Federal De Santa Catarina"
+"Brésil","Itajuba","Universidade Federal De Itajuba"
+"Brésil","João Pessoa","Universidade Federal Da Paraiba"
+"Brésil","Juiz De Fora","Universidade Federal Juiz De Fora"
+"Brésil","Lavras","Universidade Federal De Lavras (Ufla)"
+"Brésil","Maringa","Universidad Estadual De Maringa"
+"Brésil","Recife","Universidade Federal De Pernambuco"
+"Brésil","Rio De Janeiro","Universidade Federal De Rio De Janeiro"
+"Brésil","Santa Rita Do Sapucaí","Instituto Nacional De Telecommunicaçoes-Inatel"
+"Brésil","Sao Paulo","Escola Politecnica Da Universidade De Sao Paulo"
+"Brésil","Uberlandia","Universidade Federal De Uberlandia"
+"Bulgarie","Sofia","Technical University Of Sofia"
+"Bulgarie","Sofia","University Of Chemical Engineering And Mettalurgy"
+"Chili","Iquique","Universidad Arturo Prat"
+"Chili","Valparaiso","Pontificia Universidad Catolica De Valparaiso"
+"Chili","Valparaiso","Universidad De Valparaiso"
+"Chili","Valparaiso","Universidad Tecnica Federico Santa Maria"
+"Colombie","Bogota","Escuela Colombiana De Ingenieria"
+"Colombie","Bucaramanga","Universidad Autonoma De Bucaramanga"
+"Colombie","Medellin","Universidad Eafit"
+"Coree Du Sud","Daejeon","Kaist Korea Advanced Institute Of Science And Technology"
+"Coree Du Sud","Pohang","Pohang University Of Science And Technology"
+"Coree Du Sud","Seoul","Hanyang University"
+"Coree Du Sud","Seoul","Kookmin University"
+"Coree Du Sud","Seoul","Seoul National University Of Science And Technology Seoultech"
+"Equateur","Quito","Escuela Politecnica Nacional De Quito"
+"Espagne","Barcelone","Universitat Politecnica De Catalunya - Fib-Upc"
+"Espagne","Bellatera","Universidad Autonoma De Barcelona"
+"Espagne","Lleida","Universidad De Lleida"
+"Espagne","Madrid","Universidad Politecnica De Madrid (Etsiinf)"
+"Espagne","Murcia","Universidad De Murcia"
+"Espagne","Valencia","Universitat Politecnica De Valencia Etsinf"
+"Espagne","Zaragoza","Universidad De Zaragoza"
+"Finlande","Oulu","Oulu University Of Applied Sciences"
+"Finlande","Tampere","Tampere University Of Technology"
+"Islande","Reykjavik","Reykjavik University (Ru)"
+"Italie","Arcavacata","Universita Della Calabria"
+"Italie","Bologna","Universita Di Bologna"
+"Italie","Fisciano","Universita Degli Studi Di Salerno"
+"Italie","Genova","Universita Degli Studi Di Genova"
+"Italie","Milan","Politecnico Di Milano"
+"Italie","Modena","Universita Degli Studi Di Modena E Regio Emilia"
+"Italie","Napoli","Universita Degli Studi Di Napoli Federico Ii"
+"Japon","Chiba","Chiba Institute Of Technology"
+"Japon","Chiba","Chiba University"
+"Japon","Sendai","Tohoku University"
+"Liban","Beyrouth","Universite Libanaise"
+"Liban","Kaslik","Université Saint-Esprit De Kaslik"
+"Mexique","Aguascalientes","Universidad Autonoma De Aguascalientes"
+"Mexique","Ciudad Del Carmen, Campeche","Universidad Autonoma Del Carmen"
+"Mexique","Guanajuato","Universidad De Guanajuato"
+"Mexique","Merida Yucatan","Universidad Anahuac Mayab"
+"Mexique","Mexico","Instituto Politecnico Nacional"
+"Mexique","Mexico","Universidad Iberoamericana"
+"Mexique","Monterrey","Itesm - Tec De Monterrrey"
+"Mexique","Pueblar, Pue","Upaep"
+"Mexique","Zacatepec De Hidalgo","Instituto Tecnologico De Zacatepec"
+"Norvege","Trondheim","Norwegian University Of Science And Technology"
+"Pays-Bas","Enschede","University Twente"
+"Pologne","Cracovie","Agh - University Of Science And Technology"
+"Pologne","Cracovie","Politechnika Krakowska"
+"Pologne","Lodz","Lodz University Of Technology"
+"Portugal","Covilhã","Universidade Da Beira Interior"
+"République PopulaireDe Chine","Hong Kong","City University Of Hong Kong"
+"République PopulaireDe Chine","Shanghaï","Universite De Technologie Sino-Europeenne De L'Universite De Shanghai (Utseus)"
+"Republique Tcheque","Prague","Czech Technical University Of Technology"
+"Slovaquie","Kosice","Technical University Of Kosice"
+"Suede","Goteborg","Chalmers University Of Technology"
+"Suede","Lulea","Lulea University Of Technology"
+"Suisse","Lausanne","Ecole Polytechnique Federale De Lausanne"
+"Taiwan","Hsinchu","National Chiao Tung University"
+"Taiwan","Taipei","National Taiwan University Of Science And Technology"

rex/assets/destinations_extracted.csv

+"University","lat","lon"
+"Technische Universitat Ilmenau",50.6828446,10.9383699
+"Universidad Del Salvador",-34.59941195,-58.392732040256
+"Swinburne University Of Technology",-37.85240845,144.99182255
+"Technische Universitat Graz",47.06817255,15.4497318287299
+"Universiteit Gent",51.0225378,3.71068134989542
+"Universidade Federal De Minas Gerais",-19.86868095,-43.962194217906
+"Universidade Federal De Campina Grande",-7.05925115,-37.2744434555441
+"Universidade Estadual De Campinas (Unicamp)",-22.8224781,-47.0642599309425
+"Universidade Caxias Do Sul",-29.16253545,-51.1483650151909
+"Pontificia Universidade Catolica Do Parana",-23.29907785,-51.2102693622486
+"Universidade Federal Do Parana",-23.5984464,-51.6732887712978
+"Universidade Tecnologica Federal Do Parana",-25.0521636,-50.131186381692
+"Universidade Federal De Santa Catarina",-27.5823901,-48.5050400209765
+"Universidade Federal De Itajuba",-22.4137971,-45.4502726
+"Universidade Federal Da Paraiba",-6.7505593,-35.6454502176197
+"Universidade Federal Juiz De Fora",-21.7768942,-43.3691810386558
+"Universidad Estadual De Maringa",-23.77903,-53.3252683935028
+"Universidade Federal De Pernambuco",-8.05254345,-34.9510286675945
+"Universidade Federal De Rio De Janeiro",-22.2459542,-42.5232866424185
+"Escola Politecnica Da Universidade De Sao Paulo",-23.5552106,-46.7310401128074
+"Universidade Federal De Uberlandia",-18.941683,-48.2148699674138
+"Technical University Of Sofia",42.6559773,23.35512261205
+"Universidad Arturo Prat",-33.43531325,-70.662435711441
+"Pontificia Universidad Catolica De Valparaiso",-33.0449276,-71.6067964701924
+"Universidad De Valparaiso",-32.7414062,-70.7024233560544
+"Universidad Tecnica Federico Santa Maria",-33.0354207,-71.5970503
+"Escuela Colombiana De Ingenieria",4.78305195,-74.0442040297156
+"Universidad Autonoma De Bucaramanga",7.1168832,-73.1051629
+"Universidad Eafit",6.10464025,-75.4557833087589
+"Pohang University Of Science And Technology",36.0172898,129.323920997982
+"Hanyang University",37.5551257,127.0447098
+"Kookmin University",37.61147915,126.996468815557
+"Escuela Politecnica Nacional De Quito",-0.2101692,-78.4886323
+"Universidad De Murcia",38.0147121,-1.1676358
+"Universidad De Zaragoza",41.6836311,-0.885944249252812
+"Oulu University Of Applied Sciences",65.00861805,25.4726477466852
+"Tampere University Of Technology",61.44964205,23.8587746189096
+"Universita Della Calabria",39.3614022,16.2263106
+"Universita Di Bologna",44.49964345,11.354422008333
+"Universita Degli Studi Di Salerno",40.75586365,14.7804889453213
+"Universita Degli Studi Di Genova",44.4148861,8.9266707
+"Politecnico Di Milano",45.4790288,9.23221040387672
+"Universita Degli Studi Di Napoli Federico Ii",40.84526205,14.2512487738249
+"Chiba Institute Of Technology",35.6888667,140.02117559587
+"Chiba University",35.62786925,140.103465749158
+"Tohoku University",38.2530945,140.8736593
+"Universite Libanaise",33.8724848,35.5194311623001
+"Université Saint-Esprit De Kaslik",33.98168505,35.6192920532997
+"Universidad Autonoma De Aguascalientes",21.8812194,-102.2924759
+"Universidad De Guanajuato",19.7242185,-101.2440472
+"Universidad Anahuac Mayab",21.11060445,-89.6123220044136
+"Instituto Politecnico Nacional",21.1358397,-86.8493286
+"Universidad Iberoamericana",25.445315,-101.0046875
+"Upaep",19.2818385,-98.4397356
+"Instituto Tecnologico De Zacatepec",18.65203335,-99.1833987516266
+"Norwegian University Of Science And Technology",63.4103583,10.4087418620805
+"University Twente",52.2380139,6.8566761
+"Agh - University Of Science And Technology",50.0657033,19.9189586670586
+"Politechnika Krakowska",50.0717009,19.941563
+"Universidade Da Beira Interior",40.2780936,-7.50834821105263
+"City University Of Hong Kong",22.34000115,114.169702912423
+"Technical University Of Kosice",48.73280395,21.244194264458
+"Chalmers University Of Technology",57.6896523,11.9766811023544
+"Lulea University Of Technology",65.61991265,22.1410701
+"Ecole Polytechnique Federale De Lausanne",46.5186594,6.566561505148
+"National Chiao Tung University",24.78676765,120.997244116807
+"National Taiwan University Of Science And Technology",25.01350785,121.541707560048

rex/migrations/0008_auto_20180401_2253.py

+# Generated by Django 2.0.3 on 2018-04-01 20:53
+
+from django.db import migrations, models
+import rex.utils.friendly_path
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rex', '0007_auto_20180401_1806'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='university',
+            name='logo',
+            field=models.ImageField(default='path/to/my/default/image.jpg', max_length=250, upload_to=rex.utils.friendly_path.friendly_path('uploads/universities/logos/', 'name')),
+        ),
+    ]

rex/models/university/university.py

@@ -27,7 +27,8 @@ class University(models.Model):
         validators=[MinValueValidator(-180), MaxValueValidator(180)]
         )
     country = models.ForeignKey(Country, on_delete='CASCADE')
-    logo = models.ImageField(upload_to=path_and_rename, max_length=250)
+    logo = models.ImageField(upload_to=path_and_rename, max_length=250, 
+        default='path/to/my/default/image.jpg') # TODO CHANGE !
 
     def localization(self):
         return {'lat': self.localization_lat, 'lon': self.localization_lon}

rex/utils/extract_universities.py

+#!/usr/bin/env python3
+
+"""
+Script to insert the country data in the database
+
+IT HAS TO BE RUN INSIDE ./manage.py shell
+"""
+import csv 
+import os
+import time
+from geopy.geocoders import Nominatim
+
+tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations.csv')
+destinations_path = os.path.abspath(tmp)
+
+tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations_extracted.csv')
+destinations_extracted_path = os.path.abspath(tmp)
+
+if not os.path.isfile(destinations_path):
+    print(destinations_path)
+    raise Exception("Missing file containing country data")
+
+with open(destinations_path, 'rt') as input:
+    with open(destinations_extracted_path, 'w') as output:
+        print("ini")
+        reader = csv.reader(input)
+        spamwriter = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
+        geolocator = Nominatim()
+
+        failed = []
+        i = 0
+        for row in reader:
+            # handle the header
+            if i == 0:
+                header = ['University', 'lat', 'lon']
+                spamwriter.writerow(header)
+                i += 1
+            else:
+                query = row[2]#+ ', ' + row[1] + ', ' + row[0]
+                
+                while True:
+                    try:
+                        location = geolocator.geocode(query)
+                        break
+                    except:
+                        print("error during query, retrying")
+                        time.sleep(1)
+
+                print(location)
+                if location is not None:
+                    spamwriter.writerow([row[2], location.latitude, location.longitude])
+                else:
+                    failed.append(query)
+        
+        print(failed)
+
+
+            
+    
+

rex/utils/insert_universities.py

+#!/usr/bin/env python3
+
+"""
+Script to insert the country data in the database
+
+IT HAS TO BE RUN INSIDE ./manage.py shell
+"""
+import csv 
+import os
+
+from rex.models.country import Country
+from rex.models.university import University 
+
+tmp = os.path.join(os.path.realpath(__file__), '../../assets/destinations_extracted.csv')
+destinations_path = os.path.abspath(tmp)
+
+if not os.path.isfile(destinations_path):
+    print(destinations_path)
+    raise Exception("Missing file containing country data")
+
+with open(destinations_path, 'rt') as f:
+    reader = csv.reader(f)
+
+    # will be changed during university.save
+    # dumb_country =  Country.objects.get(pk='FR')
+    for row in reader:
+        if row[0] != 'University':
+            u = University(name=row[0], acronym='NULL', localization_lat=float(row[1]), localization_lon=float(row[2]))
+            u.save()