Commit cfb80538 authored by Segolene Brisemeur's avatar Segolene Brisemeur Committed by Florent Chehab

feat(external_data): load utc

* Updated models to closely match UTC db spec
* Updating loading data and tests to match
* Added UTC partner model
* Added class and command to fetch from the API on top of utc DB

Also:

* Removed department and Specialty models
* Added some denormalization and signals / custom save maintain coherence

Solves #53
WIP #28
parent 6a5274e8
Pipeline #42711 passed with stages
in 3 minutes and 51 seconds
......@@ -12,7 +12,6 @@ from backend_app.models.countryScholarship import CountryScholarship
from backend_app.models.course import Course
from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import Currency
from backend_app.models.department import Department
from backend_app.models.exchange import Exchange
from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.for_testing.moderation import ForTestingModeration
......@@ -20,7 +19,6 @@ from backend_app.models.for_testing.versioning import ForTestingVersioning
from backend_app.models.offer import Offer
from backend_app.models.pendingModeration import PendingModeration
from backend_app.models.recommendationList import RecommendationList
from backend_app.models.specialty import Specialty
from backend_app.models.university import University
from backend_app.models.universityDri import UniversityDri
from backend_app.models.universityInfo import UniversityInfo
......@@ -42,13 +40,11 @@ ALL_MODELS = [
Course,
CourseFeedback,
Currency,
Department,
Offer,
PendingModeration,
Exchange,
ExchangeFeedback,
RecommendationList,
Specialty,
University,
UniversityDri,
UniversityInfo,
......
......@@ -8,7 +8,11 @@ class BackendAppConfig(AppConfig):
from backend_app.signals.auto_creation import enable_auto_create
from backend_app.signals.squash_revisions import enable_squash_revisions
from backend_app.signals.update_nb_version import enable_update_nb_version
from backend_app.signals.coherence_external_data import (
enable_external_data_coherence,
)
enable_auto_create()
enable_squash_revisions()
enable_update_nb_version()
enable_external_data_coherence()
......@@ -4,6 +4,7 @@ from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
from backend_app.models.campus import Campus
from backend_app.models.city import City
from backend_app.models.country import Country
from backend_app.models.partner import Partner
from backend_app.models.university import University
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -33,7 +34,7 @@ class LoadUniversities(LoadGeneric):
self.add_info_and_save(city, self.admin)
univ = University.objects.update_or_create(
utc_id=row["utc_id"],
pk=row["utc_id"], # Not perfect but should do the trick
defaults={
"name": row["university"],
"acronym": row["acronym"],
......@@ -51,5 +52,15 @@ class LoadUniversities(LoadGeneric):
lat=lat,
lon=lon,
)
main_campus.save()
self.add_info_and_save(main_campus, self.admin)
Partner.objects.update_or_create(
utc_id=row["utc_id"],
defaults=dict(
univ_name=row["university"],
city=row["city"],
country=row["country"],
iso_code=row["country"],
university=univ,
),
)
......@@ -8,6 +8,7 @@ from backend_app.models.currency import Currency
from backend_app.models.exchange import Exchange
from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.language import Language
from backend_app.models.partner import Partner
from backend_app.models.taggedItems import UniversityTaggedItem
from backend_app.models.university import University
from backend_app.models.universityDri import UniversityDri
......@@ -73,20 +74,22 @@ class LoadUniversityEx(LoadGeneric):
)
self.add_info_and_save(univ_tag_1, self.admin)
EPFL_as_partner = Partner.objects.filter(university__pk=EPFL.pk)[0]
exchange1 = Exchange.objects.create(
university=EPFL,
utc_departure_id=1,
partner=EPFL_as_partner,
utc_id=1,
utc_partner_id=EPFL_as_partner.pk,
student=self.admin,
year=2019,
semester="a",
duration=1,
dual_degree=False,
double_degree=False,
master_obtained=False,
student_major="GI",
student_minor="FDD",
student_option="No",
utc_allow_courses=False,
utc_allow_login=False,
utc_allow_courses=True,
utc_allow_login=True,
)
ef = ExchangeFeedback.objects.create(
......@@ -100,19 +103,19 @@ class LoadUniversityEx(LoadGeneric):
self.add_info_and_save(ef, self.admin)
exchange2 = Exchange.objects.create(
university=EPFL,
utc_departure_id=2,
utc_partner_id=EPFL_as_partner.pk,
utc_id=2,
student=self.admin,
year=2018,
semester="a",
duration=1,
dual_degree=False,
double_degree=False,
master_obtained=False,
student_major="GI",
student_minor="FDD",
student_option="No",
utc_allow_courses=False,
utc_allow_login=True,
utc_allow_login=False,
)
ef = ExchangeFeedback.objects.create(
......@@ -125,32 +128,36 @@ class LoadUniversityEx(LoadGeneric):
)
self.add_info_and_save(ef, self.admin)
course1 = Course.objects.create(
exchange=exchange1,
utc_exchange_id=1,
course_id=1,
code="COM-401",
title="Applied data science",
link="",
nb_credit=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
)
course1 = Course.objects.update_or_create(
utc_id=1,
defaults=dict(
exchange=exchange1,
utc_exchange_id=1,
code="COM-401",
title="Applied data science",
link="",
ects=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
),
)[0]
Course.objects.create(
exchange=exchange1,
utc_exchange_id=1,
course_id=2,
code="COM-480",
title="Data vizualization",
link="",
nb_credit=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
Course.objects.update_or_create(
utc_id=2,
defaults=dict(
exchange=exchange1,
utc_exchange_id=1,
code="COM-480",
title="Data vizualization",
link="",
ects=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
),
)
cf = CourseFeedback.objects.update_or_create(
......
# Generated by Django 2.1.7 on 2019-06-29 09:22
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [("backend_app", "0002_auto_20190626_2310")]
operations = [
migrations.CreateModel(
name="Partner",
fields=[
("utc_id", models.IntegerField(primary_key=True, serialize=False)),
("univ_name", models.CharField(max_length=80)),
("address1", models.CharField(max_length=100, null=True)),
("address2", models.CharField(max_length=100, null=True)),
("zipcode", models.CharField(max_length=40, null=True)),
("city", models.CharField(max_length=40)),
("country", models.CharField(max_length=50)),
("iso_code", models.CharField(max_length=2)),
],
options={"abstract": False},
),
migrations.AlterUniqueTogether(name="specialty", unique_together=set()),
migrations.RemoveField(model_name="specialty", name="department"),
migrations.RenameField(
model_name="exchange", old_name="dual_degree", new_name="double_degree"
),
migrations.RemoveField(model_name="course", name="course_id"),
migrations.RemoveField(model_name="course", name="nb_credit"),
migrations.RemoveField(model_name="exchange", name="utc_departure_id"),
migrations.RemoveField(
model_name="offer", name="nb_seats_offered_double_degree"
),
migrations.RemoveField(model_name="offer", name="nb_seats_offered_exchange"),
migrations.RemoveField(model_name="university", name="utc_id"),
migrations.AddField(
model_name="course",
name="ects",
field=models.DecimalField(
decimal_places=2, default=0, max_digits=7, null=True
),
),
migrations.AddField(
model_name="course",
name="utc_id",
field=models.IntegerField(default=1, unique=True),
preserve_default=False,
),
migrations.AddField(
model_name="exchange",
name="utc_id",
field=models.IntegerField(default=1, unique=True),
preserve_default=False,
),
migrations.AddField(
model_name="exchange",
name="utc_partner_id",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="offer",
name="comment",
field=models.TextField(max_length=500, null=True),
),
migrations.AddField(
model_name="offer",
name="double_degree",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="offer",
name="is_master_offered",
field=models.BooleanField(default=False, null=True),
),
migrations.AddField(
model_name="offer",
name="utc_partner_id",
field=models.IntegerField(default=1),
preserve_default=False,
),
migrations.AlterField(
model_name="course",
name="code",
field=models.CharField(max_length=10, null=True),
),
migrations.AlterField(
model_name="course",
name="title",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="exchange",
name="semester",
field=models.CharField(
choices=[("a", "autumn"), ("p", "spring")],
default="a",
max_length=5,
null=True,
),
),
migrations.AlterField(
model_name="exchange",
name="student",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
migrations.AlterField(
model_name="exchange",
name="student_minor",
field=models.CharField(max_length=47, null=True),
),
migrations.AlterField(
model_name="exchange",
name="student_option",
field=models.CharField(max_length=7, null=True),
),
migrations.AlterField(
model_name="exchange",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.University",
),
),
migrations.AlterField(
model_name="exchange",
name="year",
field=models.PositiveIntegerField(default=2018, null=True),
),
migrations.AlterField(
model_name="exchangefeedback",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.University",
),
),
migrations.AlterField(
model_name="offer",
name="nb_seats_offered",
field=models.PositiveIntegerField(null=True),
),
migrations.AlterField(
model_name="offer",
name="semester",
field=models.CharField(
choices=[("a", "autumn"), ("p", "spring")],
default="a",
max_length=2,
null=True,
),
),
migrations.RemoveField(model_name="offer", name="specialties"),
migrations.AddField(
model_name="offer",
name="specialties",
field=models.CharField(max_length=4000, null=True),
),
migrations.AlterField(
model_name="offer",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.University",
),
),
migrations.AlterField(
model_name="offer",
name="year",
field=models.PositiveIntegerField(default=2018, null=True),
),
migrations.DeleteModel(name="Department"),
migrations.DeleteModel(name="Specialty"),
migrations.AddField(
model_name="partner",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
related_name="corresponding_utc_partners",
to="backend_app.University",
),
),
migrations.AddField(
model_name="exchange",
name="partner",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.Partner",
),
),
migrations.AddField(
model_name="offer",
name="partner",
field=models.ForeignKey(
default=1,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.Partner",
),
preserve_default=False,
),
]
import logging
from django.db import models
from backend_app.models.abstract.base import BaseModel
from backend_app.models.exchange import Exchange
logger = logging.getLogger("django")
class Course(BaseModel):
exchange = models.ForeignKey(
Exchange, on_delete=models.CASCADE, related_name="exchange_courses", null=True
)
utc_exchange_id = models.IntegerField()
course_id = models.IntegerField()
code = models.CharField(max_length=10)
title = models.CharField(default="", null=True, blank=True, max_length=200)
# Not using utc_id as Primary Key here for our model to be more resilient
utc_id = models.IntegerField(null=False, unique=True)
utc_exchange_id = models.IntegerField(null=False)
code = models.CharField(max_length=10, null=True)
title = models.CharField(default="", null=False, blank=True, max_length=200)
link = models.URLField(null=True, blank=True, max_length=500)
nb_credit = models.PositiveIntegerField(default=0)
ects = models.DecimalField(default=0, null=True, decimal_places=2, max_digits=7)
category = models.CharField(null=True, blank=True, max_length=5)
profile = models.CharField(null=True, blank=True, max_length=10)
tsh_profile = models.CharField(null=True, blank=True, max_length=21)
student_login = models.CharField(null=True, blank=True, max_length=8)
# a bit of denormalization
exchange = models.ForeignKey(
Exchange, on_delete=models.CASCADE, related_name="exchange_courses", null=True
)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
"""
try:
exchange = Exchange.objects.get(utc_id=self.utc_exchange_id)
self.exchange = exchange
except Exchange.DoesNotExist:
logger.warning(
"Trying to find exchange {} "
"when updating course {} but it doesn't exist".format(
self.utc_exchange_id, self.utc_id
)
)
super().save(*args, **kwargs)
from django.db import models
from backend_app.models.abstract.base import (
BaseModel,
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.permissions.app_permissions import ReadOnly
class Department(BaseModel):
code = models.CharField(primary_key=True, max_length=6)
name = models.CharField(max_length=100)
active = models.BooleanField()
class DepartmentSerializer(BaseModelSerializer):
class Meta:
model = Department
fields = "__all__"
class DepartmentViewSet(BaseModelViewSet):
queryset = Department.objects.all() # pylint: disable=E1101
serializer_class = DepartmentSerializer
permission_classes = (ReadOnly,)
end_point_route = "departments"
import logging
from django.db import models
from backend_app.models.abstract.base import BaseModel
from backend_app.models.partner import Partner
from backend_app.models.shared import SEMESTER_OPTIONS
from backend_app.models.university import University
from base_app.models import User
logger = logging.getLogger("django")
class Exchange(BaseModel):
# This model should be filled with data from the ENT
university = models.ForeignKey(University, on_delete=models.PROTECT)
utc_departure_id = models.IntegerField()
student = models.ForeignKey(User, on_delete=models.SET_NULL, null=True)
year = models.PositiveIntegerField(default=2018)
semester = models.CharField(max_length=5, choices=SEMESTER_OPTIONS, default="a")
duration = models.PositiveIntegerField()
dual_degree = models.BooleanField()
master_obtained = models.BooleanField()
student_major = models.CharField(max_length=20)
student_minor = models.CharField(max_length=7)
student_option = models.CharField(max_length=7)
utc_allow_courses = models.BooleanField()
utc_allow_login = models.BooleanField()
# Not using utc_id as Primary Key here for our model to be more resilient
utc_id = models.IntegerField(null=False, unique=True)
utc_partner_id = models.IntegerField(null=True)
year = models.PositiveIntegerField(default=2018, null=True)
semester = models.CharField(
max_length=5, choices=SEMESTER_OPTIONS, default="a", null=True
)
duration = models.PositiveIntegerField(null=False)
double_degree = models.BooleanField(null=False)
master_obtained = models.BooleanField(null=False)
student_major = models.CharField(max_length=20, null=False)
student_minor = models.CharField(max_length=47, null=True)
student_option = models.CharField(max_length=7, null=True)
utc_allow_courses = models.BooleanField(null=False)
utc_allow_login = models.BooleanField(null=False)
# a bit of denormalization
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=True)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
# (managned by signals on course save)
student = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
"""
if self.utc_partner_id is not None:
try:
self.partner = Partner.objects.get(utc_id=self.utc_partner_id)
self.university = self.partner.university
except Partner.DoesNotExist:
self.partner = None
self.university = None
logger.warning(
"Trying to find partner {}"
"when updating exchange {} but it doesn't exist".format(
self.utc_partner_id, self.utc_id
)
)
else:
self.partner = None
self.university = None
super().save(*args, **kwargs)
......@@ -16,7 +16,6 @@ from backend_app.serializers import ExchangeSerializer
class ExchangeFeedback(EssentialModule):
moderation_level = ModerationLevels.DEPENDING_ON_SITE_SETTINGS
university = models.ForeignKey(University, on_delete=models.PROTECT, default=0)
exchange = models.OneToOneField(
Exchange,
on_delete=models.CASCADE,
......@@ -36,6 +35,9 @@ class ExchangeFeedback(EssentialModule):
validators=[MinValueValidator(-5), MaxValueValidator(5)]
)
# A bit of denormalization (managed by signals)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
class ExchangeFeedbackSerializer(EssentialModuleSerializer):
exchange = ExchangeSerializer(read_only=True)
......
import logging
from django.db import models
from backend_app.models.abstract.base import (
......@@ -5,29 +7,70 @@ from backend_app.models.abstract.base import (
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.models.partner import Partner
from backend_app.models.shared import SEMESTER_OPTIONS
from backend_app.models.specialty import Specialty
from backend_app.models.university import University
from backend_app.permissions.app_permissions import ReadOnly
logger = logging.getLogger("django")
class Offer(BaseModel):
university = models.ForeignKey(University, on_delete=models.PROTECT)
year = models.PositiveIntegerField(default=2018)
semester = models.CharField(max_length=2, choices=SEMESTER_OPTIONS, default="a")
utc_partner_id = models.IntegerField(null=False)
year = models.PositiveIntegerField(default=2018, null=True)
semester = models.CharField(
max_length=2, choices=SEMESTER_OPTIONS, default="a", null=True
)
comment = models.TextField(max_length=500, null=True)
double_degree = models.BooleanField(default=False)
is_master_offered = models.BooleanField(default=False, null=True)
nb_seats_offered = models.PositiveIntegerField()
nb_seats_offered = models.PositiveIntegerField(null=True)
# null => exchange not possible
nb_seats_offered_exchange = models.PositiveIntegerField(null=True)
nb_seats_offered_double_degree = models.PositiveIntegerField(null=True)
specialties = models.ManyToManyField(Specialty, related_name="has_seats_at_univ")
specialties = models.CharField(max_length=4000, null=True)
# A bit of denormalization
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=False)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
"""
if self.utc_partner_id is not None:
try:
self.partner = Partner.objects.get(utc_id=self.utc_partner_id)
self.university = self.partner.university
except Partner.DoesNotExist:
self.partner = None
self.university = None
logger.warning(
"Trying to find partner {}"
"when updating offer {} but it doesn't exist".format(
self.utc_partner_id, self.pk
)
)
else:
self.partner = None
self.university = None
super().save(*args, **kwargs)
class OfferSerializer(BaseModelSerializer):
class Meta:
model = Offer
fields = "__all__"
fields = BaseModelSerializer.Meta.fields + (
"year",
"semester",
"commend",
"double_degree",
"is_master_offered",
"specialties",
"partner",
"university",
)
class OfferViewSet(BaseModelViewSet):
......
from django.db import models
from backend_app.models.abstract.base import BaseModel
from backend_app.models.university import University
class Partner(BaseModel):
"""
Model that stores raw information from the UTC database regarding university partners.
Some universities in UTC database are present multiple times. To accommodate for this,