Commit 0e4c1aef authored by Florent Chehab's avatar Florent Chehab Committed by Florent Chehab
Browse files

removed(all TODOs):

Some of them were simply removed, oter were fixed.
From now on, the use of TODO is not permitted in the code.
parent 5530363c
Pipeline #36624 passed with stages
in 4 minutes and 54 seconds
[flake8]
# E501 line too long
ignore = E501,W503
exclude =
env/*
node_modules/*
base_app/settings/development.py
from .load_all import load_all
# We firest need to define ASSETS_PATH to prevent cyclic imports
from os import path
__all__ = ["load_all"]
ASSETS_PATH = path.join(path.realpath(__file__), "../assets/") # noqa: E402
from .load_all import load_all # noqa: E402
__all__ = ["load_all", "ASSETS_PATH"]
"Pays","Ville","Nom de l'établissement"
"Allemagne","Hamburg","Technische Universitat Hamburg"
"Allemagne","Ilmenau","Technische Universitat Ilmenau"
"Argentine","Buenos Aires","Universidad Del Salvador"
"Australie","Victoria","Swinburne University Of Technology"
"Autriche","Graz","Technische Universitat Graz"
"Belgique","Gent","Universiteit Gent"
"Brésil","Belo Horizonte","Universidade Federal De Minas Gerais"
"Brésil","Campina Grande","Universidade Federal De Campina Grande"
"Brésil","Campinas-Sao Paulo","Universidade Estadual De Campinas (Unicamp)"
"Brésil","Caxias Do Sul","Universidade Caxias Do Sul"
"Brésil","Curitiba","Pontificia Universidade Catolica Do Parana"
"Brésil","Curitiba","Universidade Federal Do Parana"
"Brésil","Curitiba","Universidade Tecnologica Federal Do Parana"
"Brésil","Florianópolis","Universidade Federal De Santa Catarina"
"Brésil","Itajuba","Universidade Federal De Itajuba"
"Brésil","João Pessoa","Universidade Federal Da Paraiba"
"Brésil","Juiz De Fora","Universidade Federal Juiz De Fora"
"Brésil","Lavras","Universidade Federal De Lavras (Ufla)"
"Brésil","Maringa","Universidad Estadual De Maringa"
"Brésil","Recife","Universidade Federal De Pernambuco"
"Brésil","Rio De Janeiro","Universidade Federal De Rio De Janeiro"
"Brésil","Santa Rita Do Sapucaí","Instituto Nacional De Telecommunicaçoes-Inatel"
"Brésil","Sao Paulo","Escola Politecnica Da Universidade De Sao Paulo"
"Brésil","Uberlandia","Universidade Federal De Uberlandia"
"Bulgarie","Sofia","Technical University Of Sofia"
"Bulgarie","Sofia","University Of Chemical Engineering And Mettalurgy"
"Chili","Iquique","Universidad Arturo Prat"
"Chili","Valparaiso","Pontificia Universidad Catolica De Valparaiso"
"Chili","Valparaiso","Universidad De Valparaiso"
"Chili","Valparaiso","Universidad Tecnica Federico Santa Maria"
"Colombie","Bogota","Escuela Colombiana De Ingenieria"
"Colombie","Bucaramanga","Universidad Autonoma De Bucaramanga"
"Colombie","Medellin","Universidad Eafit"
"Coree Du Sud","Daejeon","Kaist Korea Advanced Institute Of Science And Technology"
"Coree Du Sud","Pohang","Pohang University Of Science And Technology"
"Coree Du Sud","Seoul","Hanyang University"
"Coree Du Sud","Seoul","Kookmin University"
"Coree Du Sud","Seoul","Seoul National University Of Science And Technology Seoultech"
"Equateur","Quito","Escuela Politecnica Nacional De Quito"
"Espagne","Barcelone","Universitat Politecnica De Catalunya - Fib-Upc"
"Espagne","Bellatera","Universidad Autonoma De Barcelona"
"Espagne","Lleida","Universidad De Lleida"
"Espagne","Madrid","Universidad Politecnica De Madrid (Etsiinf)"
"Espagne","Murcia","Universidad De Murcia"
"Espagne","Valencia","Universitat Politecnica De Valencia Etsinf"
"Espagne","Zaragoza","Universidad De Zaragoza"
"Finlande","Oulu","Oulu University Of Applied Sciences"
"Finlande","Tampere","Tampere University Of Technology"
"Islande","Reykjavik","Reykjavik University (Ru)"
"Italie","Arcavacata","Universita Della Calabria"
"Italie","Bologna","Universita Di Bologna"
"Italie","Fisciano","Universita Degli Studi Di Salerno"
"Italie","Genova","Universita Degli Studi Di Genova"
"Italie","Milan","Politecnico Di Milano"
"Italie","Modena","Universita Degli Studi Di Modena E Regio Emilia"
"Italie","Napoli","Universita Degli Studi Di Napoli Federico Ii"
"Japon","Chiba","Chiba Institute Of Technology"
"Japon","Chiba","Chiba University"
"Japon","Sendai","Tohoku University"
"Liban","Beyrouth","Universite Libanaise"
"Liban","Kaslik","Université Saint-Esprit De Kaslik"
"Mexique","Aguascalientes","Universidad Autonoma De Aguascalientes"
"Mexique","Ciudad Del Carmen, Campeche","Universidad Autonoma Del Carmen"
"Mexique","Guanajuato","Universidad De Guanajuato"
"Mexique","Merida Yucatan","Universidad Anahuac Mayab"
"Mexique","Mexico","Instituto Politecnico Nacional"
"Mexique","Mexico","Universidad Iberoamericana"
"Mexique","Monterrey","Itesm - Tec De Monterrrey"
"Mexique","Pueblar, Pue","Upaep"
"Mexique","Zacatepec De Hidalgo","Instituto Tecnologico De Zacatepec"
"Norvege","Trondheim","Norwegian University Of Science And Technology"
"Pays-Bas","Enschede","University Twente"
"Pologne","Cracovie","Agh - University Of Science And Technology"
"Pologne","Cracovie","Politechnika Krakowska"
"Pologne","Lodz","Lodz University Of Technology"
"Portugal","Covilhã","Universidade Da Beira Interior"
"République PopulaireDe Chine","Hong Kong","City University Of Hong Kong"
"République PopulaireDe Chine","Shanghaï","Universite De Technologie Sino-Europeenne De L'Universite De Shanghai (Utseus)"
"Republique Tcheque","Prague","Czech Technical University Of Technology"
"Slovaquie","Kosice","Technical University Of Kosice"
"Suede","Goteborg","Chalmers University Of Technology"
"Suede","Lulea","Lulea University Of Technology"
"Suisse","Lausanne","Ecole Polytechnique Federale De Lausanne"
"Taiwan","Hsinchu","National Chiao Tung University"
"Taiwan","Taipei","National Taiwan University Of Science And Technology"
"""
Script to insert the country data in the database
IT HAS TO BE RUN INSIDE ./manage.py shell
TODO YURK. Use pandas @florent !!
"""
import csv
import os
import time
import reverse_geocoder as rg
from geopy.geocoders import Nominatim
tmp = os.path.join(os.path.realpath(__file__), "../../assets/destinations.csv")
destinations_path = os.path.abspath(tmp)
tmp = os.path.join(
os.path.realpath(__file__), "../../assets/destinations_extracted.csv"
)
destinations_extracted_path = os.path.abspath(tmp)
if not os.path.isfile(destinations_path):
print(destinations_path)
raise Exception("Missing file containing country data")
with open(destinations_path, "rt") as input:
with open(destinations_extracted_path, "w") as output:
print("ini")
reader = csv.reader(input)
spamwriter = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
geolocator = Nominatim()
failed = []
i = 0
for row in reader:
# handle the header
if i == 0:
header = ["university", "city", "country", "lat", "lon"]
spamwriter.writerow(header)
i += 1
else:
query = row[2] # + ', ' + row[1] + ', ' + row[0]
while True:
try:
location = geolocator.geocode(query)
break
except: # noqa: E722
print("error during query, retrying")
time.sleep(0.5)
if location is not None:
coord = (location.latitude, location.longitude)
res = rg.search(coord, verbose=False)
line = [
row[2],
row[1],
res[0]["cc"],
location.latitude,
location.longitude,
]
print(line)
spamwriter.writerow(line)
else:
failed.append(query)
print(failed)
......@@ -6,6 +6,7 @@ from .loadTags import LoadTags
from .loadCurrencies import LoadCurrencies
from .loadUniversityEx import LoadUniversityEx
__all__ = [
"LoadGroups",
"LoadAdminUser",
......@@ -15,5 +16,3 @@ __all__ = [
"LoadCurrencies",
"LoadUniversityEx",
]
# TODO in all loading files, use a path to the assets folder define somewhere else.
......@@ -6,7 +6,7 @@ class LoadAdminUser(object):
def __init__(self):
"""
Creating admin user by creating a user and setting is_staff is True
didn't enable access to the admin system for an unkown reason.
didn't enable access to the admin system for an unknown reason.
Need to use create_superuser.
"""
user = User.objects.filter(username="admin")
......
import os
from base_app.models import User
from os.path import abspath, join
import pandas as pd
from backend_app.load_data import ASSETS_PATH
from backend_app.models.country import Country
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -17,12 +17,8 @@ class LoadCountries(LoadGeneric):
self.admin = admin
def load(self):
tmp = os.path.join(os.path.realpath(__file__), "../../assets/country.csv")
country_file_loc = os.path.abspath(tmp)
tmp = os.path.join(
os.path.realpath(__file__), "../../assets/alpha-conv-table.csv"
)
conv_alpha_file_loc = os.path.abspath(tmp)
country_file_loc = abspath(join(ASSETS_PATH, "country.csv"))
conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv"))
country_pd = pd.read_csv(
country_file_loc, sep=",", header=0, dtype=object
......
import csv
import os
from decimal import Decimal
from os.path import abspath, join
from base_app.models import User
from backend_app.load_data import ASSETS_PATH
from backend_app.models.currency import Currency
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -18,8 +18,7 @@ class LoadCurrencies(LoadGeneric):
self.admin = admin
def load(self):
tmp = os.path.join(os.path.realpath(__file__), "../../assets/currencies.csv")
currencies_file_loc = os.path.abspath(tmp)
currencies_file_loc = abspath(join(ASSETS_PATH, "currencies.csv"))
with open(currencies_file_loc) as csvfile:
reader = csv.reader(csvfile, quotechar='"')
......
import os
from base_app.models import User
from os.path import abspath, join
import pandas as pd
from backend_app.load_data import ASSETS_PATH
from backend_app.models.campus import Campus
from backend_app.models.city import City
from backend_app.models.country import Country
from backend_app.models.university import University
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -20,11 +20,7 @@ class LoadUniversities(LoadGeneric):
self.admin = admin
def load(self):
tmp = os.path.join(
os.path.realpath(__file__), "../../assets/destinations_extracted.csv"
)
destinations_path = os.path.abspath(tmp)
destinations_path = abspath(join(ASSETS_PATH, "destinations_extracted.csv"))
data = pd.read_csv(destinations_path, sep=",", header=0, dtype=object).fillna(
""
......
......@@ -57,7 +57,7 @@ class LoadUniversityEx(LoadGeneric):
country_scholarship = CountryScholarship(
title="Swiss European Mobility Programme",
type="Bourse du gouvernement suisse",
short_description="Bourse du gouvernement suisse",
currency=CHF,
frequency="s",
amount_min=2200,
......
......@@ -11,1025 +11,2122 @@ import django.db.models.deletion
class Migration(migrations.Migration):
replaces = [('backend_app', '0001_initial'), ('backend_app', '0002_auto_20190310_1726'), ('backend_app', '0003_auto_20190310_1731'), ('backend_app', '0004_auto_20190310_1756')]
replaces = [
("backend_app", "0001_initial"),
("backend_app", "0002_auto_20190310_1726"),
("backend_app", "0003_auto_20190310_1731"),
("backend_app", "0004_auto_20190310_1756"),
]
initial = True
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
('base_app', '0001_initial'),
("contenttypes", "0002_remove_content_type_name"),
("base_app", "0001_initial"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Campus',
name="Campus",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('updated_on', models.DateTimeField(null=True)),
('moderated_on', models.DateTimeField(null=True)),
('obj_moderation_level', models.SmallIntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), backend_app.models.abstract.my_model.myModel.validate_obj_model_lv])),
('title', models.CharField(blank=True, default='', max_length=150)),
('comment', models.CharField(blank=True, default='', max_length=5000)),
('useful_links', backend_app.fields.JSONField(default=list)),
('importance_level', models.CharField(choices=[('-', 'normal'), ('+', 'important'), ('++', 'IMPORTANT')], default='-', max_length=2)),
('is_main_campus', models.BooleanField()),
('name', models.CharField(blank=True, default='', max_length=200)),
('lat', models.DecimalField(decimal_places=6, max_digits=10, validators=[django.core.validators.MinValueValidator(-85.05112878), django.core.validators.MaxValueValidator(85.05112878)])),
('lon', models.DecimalField(decimal_places=6, max_digits=10, validators=[django.core.validators.MinValueValidator(-180), django.core.validators.MaxValueValidator(180)])),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("updated_on", models.DateTimeField(null=True)),
("moderated_on", models.DateTimeField(null=True)),
(
"obj_moderation_level",
models.SmallIntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(0),
backend_app.models.abstract.my_model.myModel.validate_obj_model_lv,
],
),
),
("title", models.CharField(blank=True, default="", max_length=150)),
("comment", models.CharField(blank=True, default="", max_length=5000)),
("useful_links", backend_app.fields.JSONField(default=list)),
(
"importance_level",
models.CharField(
choices=[
("-", "normal"),
("+", "important"),
("++", "IMPORTANT"),
],
default="-",
max_length=2,
),
),
("is_main_campus", models.BooleanField()),
("name", models.CharField(blank=True, default="", max_length=200)),
(
"lat",
models.DecimalField(
decimal_places=6,
max_digits=10,
validators=[
django.core.validators.MinValueValidator(-85.05112878),
django.core.validators.MaxValueValidator(85.05112878),
],
),
),
(
"lon",
models.DecimalField(
decimal_places=6,
max_digits=10,
validators=[
django.core.validators.MinValueValidator(-180),
django.core.validators.MaxValueValidator(180),
],
),
),
],
),
migrations.CreateModel(
name='CampusTaggedItem',
name="CampusTaggedItem",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('updated_on', models.DateTimeField(null=True)),
('moderated_on', models.DateTimeField(null=True)),
('obj_moderation_level', models.SmallIntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), backend_app.models.abstract.my_model.myModel.validate_obj_model_lv])),
('title', models.CharField(blank=True, default='', max_length=150)),
('comment', models.CharField(blank=True, default='', max_length=5000)),
('useful_links', backend_app.fields.JSONField(default=list)),
('importance_level', models.CharField(choices=[('-', 'normal'), ('+', 'important'), ('++', 'IMPORTANT')], default='-', max_length=2)),
('custom_content', backend_app.fields.JSONField(default=dict)),
('campus', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='campus_tagged_items', to='backend_app.Campus')),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("updated_on", models.DateTimeField(null=True)),
("moderated_on", models.DateTimeField(null=True)),
(
"obj_moderation_level",
models.SmallIntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(0),
backend_app.models.abstract.my_model.myModel.validate_obj_model_lv,
],
),
),
("title", models.CharField(blank=True, default="", max_length=150)),
("comment", models.CharField(blank=True, default="", max_length=5000)),
("useful_links", backend_app.fields.JSONField(default=list)),
(
"importance_level",
models.CharField(
choices=[
("-", "normal"),
("+", "important"),
("++", "IMPORTANT"),
],
default="-",
max_length=2,
),
),
("custom_content", backend_app.fields.JSONField(default=dict)),
(
"campus",
models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="campus_tagged_items",
to="backend_app.Campus",
),
),
],
),
migrations.CreateModel(
name='City',
name="City",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('updated_on', models.DateTimeField(null=True)),
('moderated_on', models.DateTimeField(null=True)),
('obj_moderation_level', models.SmallIntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), backend_app.models.abstract.my_model.myModel.validate_obj_model_lv])),
('name', models.CharField(max_length=200)),
('local_name', models.CharField(blank=True, default='', max_length=200)),
('area', models.CharField(blank=True, default='', max_length=200)),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("updated_on", models.DateTimeField(null=True)),
("moderated_on", models.DateTimeField(null=True)),
(
"obj_moderation_level",
models.SmallIntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(0),
backend_app.models.abstract.my_model.myModel.validate_obj_model_lv,
],
),
),
("name", models.CharField(max_length=200)),
(
"local_name",
models.CharField(blank=True, default="", max_length=200),
),
("area", models.CharField(blank=True, default="", max_length=200)),
],
options={
'abstract': False,
},
options={"abstract": False},
),
migrations.CreateModel(
name='CityTaggedItem',
name="CityTaggedItem",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('updated_on', models.DateTimeField(null=True)),
('moderated_on', models.DateTimeField(null=True)),
('obj_moderation_level', models.SmallIntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), backend_app.models.abstract.my_model.myModel.validate_obj_model_lv])),
('title', models.CharField(blank=True, default='', max_length=150)),
('comment', models.CharField(blank=True, default='', max_length=5000)),
('useful_links', backend_app.fields.JSONField(default=list)),
('importance_level', models.CharField(choices=[('-', 'normal'), ('+', 'important'), ('++', 'IMPORTANT')], default='-', max_length=2)),
('custom_content', backend_app.fields.JSONField(default=dict)),
('city', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='city_tagged_items', to='backend_app.City')),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("updated_on", models.DateTimeField(null=True)),
("moderated_on", models.DateTimeField(null=True)),
(
"obj_moderation_level",
models.SmallIntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(0),
backend_app.models.abstract.my_model.myModel.validate_obj_model_lv,
],
),
),
("title", models.CharField(blank=True, default="", max_length=150)),
("comment", models.CharField(blank=True, default="", max_length=5000)),
("useful_links", backend_app.fields.JSONField(default=list)),
(
"importance_level",
models.CharField(
choices=[
("-", "normal"),
("+", "important"),
("++", "IMPORTANT"),
],
default="-",
max_length=2,
),
),
("custom_content", backend_app.fields.JSONField(default=dict)),
(
"city",
models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="city_tagged_items",
to="backend_app.City",
),
),
],
),
migrations.CreateModel(
name='Country',
name="Country",
fields=[
('updated_on', models.DateTimeField(null=True)),
('moderated_on', models.DateTimeField(null=True)),
('obj_moderation_level', models.SmallIntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), backend_app.models.abstract.my_model.myModel.validate_obj_model_lv])),
('name', models.CharField(max_length=200)),