Commit cfb80538 authored by Segolene Brisemeur's avatar Segolene Brisemeur Committed by Florent Chehab

feat(external_data): load utc

* Updated models to closely match UTC db spec
* Updating loading data and tests to match
* Added UTC partner model
* Added class and command to fetch from the API on top of utc DB

Also:

* Removed department and Specialty models
* Added some denormalization and signals / custom save maintain coherence

Solves #53
WIP #28
parent 6a5274e8
Pipeline #42711 passed with stages
in 3 minutes and 51 seconds
......@@ -12,7 +12,6 @@ from backend_app.models.countryScholarship import CountryScholarship
from backend_app.models.course import Course
from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import Currency
from backend_app.models.department import Department
from backend_app.models.exchange import Exchange
from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.for_testing.moderation import ForTestingModeration
......@@ -20,7 +19,6 @@ from backend_app.models.for_testing.versioning import ForTestingVersioning
from backend_app.models.offer import Offer
from backend_app.models.pendingModeration import PendingModeration
from backend_app.models.recommendationList import RecommendationList
from backend_app.models.specialty import Specialty
from backend_app.models.university import University
from backend_app.models.universityDri import UniversityDri
from backend_app.models.universityInfo import UniversityInfo
......@@ -42,13 +40,11 @@ ALL_MODELS = [
Course,
CourseFeedback,
Currency,
Department,
Offer,
PendingModeration,
Exchange,
ExchangeFeedback,
RecommendationList,
Specialty,
University,
UniversityDri,
UniversityInfo,
......
......@@ -8,7 +8,11 @@ class BackendAppConfig(AppConfig):
from backend_app.signals.auto_creation import enable_auto_create
from backend_app.signals.squash_revisions import enable_squash_revisions
from backend_app.signals.update_nb_version import enable_update_nb_version
from backend_app.signals.coherence_external_data import (
enable_external_data_coherence,
)
enable_auto_create()
enable_squash_revisions()
enable_update_nb_version()
enable_external_data_coherence()
......@@ -4,6 +4,7 @@ from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
from backend_app.models.campus import Campus
from backend_app.models.city import City
from backend_app.models.country import Country
from backend_app.models.partner import Partner
from backend_app.models.university import University
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -33,7 +34,7 @@ class LoadUniversities(LoadGeneric):
self.add_info_and_save(city, self.admin)
univ = University.objects.update_or_create(
utc_id=row["utc_id"],
pk=row["utc_id"], # Not perfect but should do the trick
defaults={
"name": row["university"],
"acronym": row["acronym"],
......@@ -51,5 +52,15 @@ class LoadUniversities(LoadGeneric):
lat=lat,
lon=lon,
)
main_campus.save()
self.add_info_and_save(main_campus, self.admin)
Partner.objects.update_or_create(
utc_id=row["utc_id"],
defaults=dict(
univ_name=row["university"],
city=row["city"],
country=row["country"],
iso_code=row["country"],
university=univ,
),
)
......@@ -8,6 +8,7 @@ from backend_app.models.currency import Currency
from backend_app.models.exchange import Exchange
from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.language import Language
from backend_app.models.partner import Partner
from backend_app.models.taggedItems import UniversityTaggedItem
from backend_app.models.university import University
from backend_app.models.universityDri import UniversityDri
......@@ -73,20 +74,22 @@ class LoadUniversityEx(LoadGeneric):
)
self.add_info_and_save(univ_tag_1, self.admin)
EPFL_as_partner = Partner.objects.filter(university__pk=EPFL.pk)[0]
exchange1 = Exchange.objects.create(
university=EPFL,
utc_departure_id=1,
partner=EPFL_as_partner,
utc_id=1,
utc_partner_id=EPFL_as_partner.pk,
student=self.admin,
year=2019,
semester="a",
duration=1,
dual_degree=False,
double_degree=False,
master_obtained=False,
student_major="GI",
student_minor="FDD",
student_option="No",
utc_allow_courses=False,
utc_allow_login=False,
utc_allow_courses=True,
utc_allow_login=True,
)
ef = ExchangeFeedback.objects.create(
......@@ -100,19 +103,19 @@ class LoadUniversityEx(LoadGeneric):
self.add_info_and_save(ef, self.admin)
exchange2 = Exchange.objects.create(
university=EPFL,
utc_departure_id=2,
utc_partner_id=EPFL_as_partner.pk,
utc_id=2,
student=self.admin,
year=2018,
semester="a",
duration=1,
dual_degree=False,
double_degree=False,
master_obtained=False,
student_major="GI",
student_minor="FDD",
student_option="No",
utc_allow_courses=False,
utc_allow_login=True,
utc_allow_login=False,
)
ef = ExchangeFeedback.objects.create(
......@@ -125,32 +128,36 @@ class LoadUniversityEx(LoadGeneric):
)
self.add_info_and_save(ef, self.admin)
course1 = Course.objects.create(
exchange=exchange1,
utc_exchange_id=1,
course_id=1,
code="COM-401",
title="Applied data science",
link="",
nb_credit=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
)
course1 = Course.objects.update_or_create(
utc_id=1,
defaults=dict(
exchange=exchange1,
utc_exchange_id=1,
code="COM-401",
title="Applied data science",
link="",
ects=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
),
)[0]
Course.objects.create(
exchange=exchange1,
utc_exchange_id=1,
course_id=2,
code="COM-480",
title="Data vizualization",
link="",
nb_credit=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
Course.objects.update_or_create(
utc_id=2,
defaults=dict(
exchange=exchange1,
utc_exchange_id=1,
code="COM-480",
title="Data vizualization",
link="",
ects=5,
category="TM",
profile="PSF",
tsh_profile="",
student_login="admin",
),
)
cf = CourseFeedback.objects.update_or_create(
......
# Generated by Django 2.1.7 on 2019-06-29 09:22
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [("backend_app", "0002_auto_20190626_2310")]
operations = [
migrations.CreateModel(
name="Partner",
fields=[
("utc_id", models.IntegerField(primary_key=True, serialize=False)),
("univ_name", models.CharField(max_length=80)),
("address1", models.CharField(max_length=100, null=True)),
("address2", models.CharField(max_length=100, null=True)),
("zipcode", models.CharField(max_length=40, null=True)),
("city", models.CharField(max_length=40)),
("country", models.CharField(max_length=50)),
("iso_code", models.CharField(max_length=2)),
],
options={"abstract": False},
),
migrations.AlterUniqueTogether(name="specialty", unique_together=set()),
migrations.RemoveField(model_name="specialty", name="department"),
migrations.RenameField(
model_name="exchange", old_name="dual_degree", new_name="double_degree"
),
migrations.RemoveField(model_name="course", name="course_id"),
migrations.RemoveField(model_name="course", name="nb_credit"),
migrations.RemoveField(model_name="exchange", name="utc_departure_id"),
migrations.RemoveField(
model_name="offer", name="nb_seats_offered_double_degree"
),
migrations.RemoveField(model_name="offer", name="nb_seats_offered_exchange"),
migrations.RemoveField(model_name="university", name="utc_id"),
migrations.AddField(
model_name="course",
name="ects",
field=models.DecimalField(
decimal_places=2, default=0, max_digits=7, null=True
),
),
migrations.AddField(
model_name="course",
name="utc_id",
field=models.IntegerField(default=1, unique=True),
preserve_default=False,
),
migrations.AddField(
model_name="exchange",
name="utc_id",
field=models.IntegerField(default=1, unique=True),
preserve_default=False,
),
migrations.AddField(
model_name="exchange",
name="utc_partner_id",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="offer",
name="comment",
field=models.TextField(max_length=500, null=True),
),
migrations.AddField(
model_name="offer",
name="double_degree",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="offer",
name="is_master_offered",
field=models.BooleanField(default=False, null=True),
),
migrations.AddField(
model_name="offer",
name="utc_partner_id",
field=models.IntegerField(default=1),
preserve_default=False,
),
migrations.AlterField(
model_name="course",
name="code",
field=models.CharField(max_length=10, null=True),
),
migrations.AlterField(
model_name="course",
name="title",
field=models.CharField(blank=True, default="", max_length=200),
),
migrations.AlterField(
model_name="exchange",
name="semester",
field=models.CharField(
choices=[("a", "autumn"), ("p", "spring")],
default="a",
max_length=5,
null=True,
),
),
migrations.AlterField(
model_name="exchange",
name="student",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
migrations.AlterField(
model_name="exchange",
name="student_minor",
field=models.CharField(max_length=47, null=True),
),
migrations.AlterField(
model_name="exchange",
name="student_option",
field=models.CharField(max_length=7, null=True),
),
migrations.AlterField(
model_name="exchange",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.University",
),
),
migrations.AlterField(
model_name="exchange",
name="year",
field=models.PositiveIntegerField(default=2018, null=True),
),
migrations.AlterField(
model_name="exchangefeedback",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.University",
),
),
migrations.AlterField(
model_name="offer",
name="nb_seats_offered",
field=models.PositiveIntegerField(null=True),
),
migrations.AlterField(
model_name="offer",
name="semester",
field=models.CharField(
choices=[("a", "autumn"), ("p", "spring")],
default="a",
max_length=2,
null=True,
),
),
migrations.RemoveField(model_name="offer", name="specialties"),
migrations.AddField(
model_name="offer",
name="specialties",
field=models.CharField(max_length=4000, null=True),
),
migrations.AlterField(
model_name="offer",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.University",
),
),
migrations.AlterField(
model_name="offer",
name="year",
field=models.PositiveIntegerField(default=2018, null=True),
),
migrations.DeleteModel(name="Department"),
migrations.DeleteModel(name="Specialty"),
migrations.AddField(
model_name="partner",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
related_name="corresponding_utc_partners",
to="backend_app.University",
),
),
migrations.AddField(
model_name="exchange",
name="partner",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.Partner",
),
),
migrations.AddField(
model_name="offer",
name="partner",
field=models.ForeignKey(
default=1,
on_delete=django.db.models.deletion.PROTECT,
to="backend_app.Partner",
),
preserve_default=False,
),
]
import logging
from django.db import models
from backend_app.models.abstract.base import BaseModel
from backend_app.models.exchange import Exchange
logger = logging.getLogger("django")
class Course(BaseModel):
exchange = models.ForeignKey(
Exchange, on_delete=models.CASCADE, related_name="exchange_courses", null=True
)
utc_exchange_id = models.IntegerField()
course_id = models.IntegerField()
code = models.CharField(max_length=10)
title = models.CharField(default="", null=True, blank=True, max_length=200)
# Not using utc_id as Primary Key here for our model to be more resilient
utc_id = models.IntegerField(null=False, unique=True)
utc_exchange_id = models.IntegerField(null=False)
code = models.CharField(max_length=10, null=True)
title = models.CharField(default="", null=False, blank=True, max_length=200)
link = models.URLField(null=True, blank=True, max_length=500)
nb_credit = models.PositiveIntegerField(default=0)
ects = models.DecimalField(default=0, null=True, decimal_places=2, max_digits=7)
category = models.CharField(null=True, blank=True, max_length=5)
profile = models.CharField(null=True, blank=True, max_length=10)
tsh_profile = models.CharField(null=True, blank=True, max_length=21)
student_login = models.CharField(null=True, blank=True, max_length=8)
# a bit of denormalization
exchange = models.ForeignKey(
Exchange, on_delete=models.CASCADE, related_name="exchange_courses", null=True
)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
"""
try:
exchange = Exchange.objects.get(utc_id=self.utc_exchange_id)
self.exchange = exchange
except Exchange.DoesNotExist:
logger.warning(
"Trying to find exchange {} "
"when updating course {} but it doesn't exist".format(
self.utc_exchange_id, self.utc_id
)
)
super().save(*args, **kwargs)
from django.db import models
from backend_app.models.abstract.base import (
BaseModel,
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.permissions.app_permissions import ReadOnly
class Department(BaseModel):
code = models.CharField(primary_key=True, max_length=6)
name = models.CharField(max_length=100)
active = models.BooleanField()
class DepartmentSerializer(BaseModelSerializer):
class Meta:
model = Department
fields = "__all__"
class DepartmentViewSet(BaseModelViewSet):
queryset = Department.objects.all() # pylint: disable=E1101
serializer_class = DepartmentSerializer
permission_classes = (ReadOnly,)
end_point_route = "departments"
import logging
from django.db import models
from backend_app.models.abstract.base import BaseModel
from backend_app.models.partner import Partner
from backend_app.models.shared import SEMESTER_OPTIONS
from backend_app.models.university import University
from base_app.models import User
logger = logging.getLogger("django")
class Exchange(BaseModel):
# This model should be filled with data from the ENT
university = models.ForeignKey(University, on_delete=models.PROTECT)
utc_departure_id = models.IntegerField()
student = models.ForeignKey(User, on_delete=models.SET_NULL, null=True)
year = models.PositiveIntegerField(default=2018)
semester = models.CharField(max_length=5, choices=SEMESTER_OPTIONS, default="a")
duration = models.PositiveIntegerField()
dual_degree = models.BooleanField()
master_obtained = models.BooleanField()
student_major = models.CharField(max_length=20)
student_minor = models.CharField(max_length=7)
student_option = models.CharField(max_length=7)
utc_allow_courses = models.BooleanField()
utc_allow_login = models.BooleanField()
# Not using utc_id as Primary Key here for our model to be more resilient
utc_id = models.IntegerField(null=False, unique=True)
utc_partner_id = models.IntegerField(null=True)
year = models.PositiveIntegerField(default=2018, null=True)
semester = models.CharField(
max_length=5, choices=SEMESTER_OPTIONS, default="a", null=True
)
duration = models.PositiveIntegerField(null=False)
double_degree = models.BooleanField(null=False)
master_obtained = models.BooleanField(null=False)
student_major = models.CharField(max_length=20, null=False)
student_minor = models.CharField(max_length=47, null=True)
student_option = models.CharField(max_length=7, null=True)
utc_allow_courses = models.BooleanField(null=False)
utc_allow_login = models.BooleanField(null=False)
# a bit of denormalization
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=True)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
# (managned by signals on course save)
student = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
"""
if self.utc_partner_id is not None:
try:
self.partner = Partner.objects.get(utc_id=self.utc_partner_id)
self.university = self.partner.university
except Partner.DoesNotExist:
self.partner = None
self.university = None
logger.warning(
"Trying to find partner {}"
"when updating exchange {} but it doesn't exist".format(
self.utc_partner_id, self.utc_id
)
)
else:
self.partner = None
self.university = None
super().save(*args, **kwargs)
......@@ -16,7 +16,6 @@ from backend_app.serializers import ExchangeSerializer
class ExchangeFeedback(EssentialModule):
moderation_level = ModerationLevels.DEPENDING_ON_SITE_SETTINGS
university = models.ForeignKey(University, on_delete=models.PROTECT, default=0)
exchange = models.OneToOneField(
Exchange,
on_delete=models.CASCADE,
......@@ -36,6 +35,9 @@ class ExchangeFeedback(EssentialModule):
validators=[MinValueValidator(-5), MaxValueValidator(5)]
)
# A bit of denormalization (managed by signals)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
class ExchangeFeedbackSerializer(EssentialModuleSerializer):
exchange = ExchangeSerializer(read_only=True)
......
import logging
from django.db import models
from backend_app.models.abstract.base import (
......@@ -5,29 +7,70 @@ from backend_app.models.abstract.base import (
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.models.partner import Partner
from backend_app.models.shared import SEMESTER_OPTIONS
from backend_app.models.specialty import Specialty
from backend_app.models.university import University
from backend_app.permissions.app_permissions import ReadOnly
logger = logging.getLogger("django")
class Offer(BaseModel):
university = models.ForeignKey(University, on_delete=models.PROTECT)
year = models.PositiveIntegerField(default=2018)
semester = models.CharField(max_length=2, choices=SEMESTER_OPTIONS, default="a")
utc_partner_id = models.IntegerField(null=False)
year = models.PositiveIntegerField(default=2018, null=True)
semester = models.CharField(
max_length=2, choices=SEMESTER_OPTIONS, default="a", null=True
)
comment = models.TextField(max_length=500, null=True)
double_degree = models.BooleanField(default=False)
is_master_offered = models.BooleanField(default=False, null=True)
nb_seats_offered = models.PositiveIntegerField()
nb_seats_offered = models.PositiveIntegerField(null=True)
# null => exchange not possible
nb_seats_offered_exchange = models.PositiveIntegerField(null=True)
nb_seats_offered_double_degree = models.PositiveIntegerField(null=True)
specialties = models.ManyToManyField(Specialty, related_name="has_seats_at_univ")
specialties = models.CharField(max_length=4000, null=True)
# A bit of denormalization
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=False)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
"""
if self.utc_partner_id is not None:
try:
self.partner = Partner.objects.get(utc_id=self.utc_partner_id)
self.university = self.partner.university
except Partner.DoesNotExist:
self.partner = None
self.university = None
logger.warning(
"Trying to find partner {}"
"when updating offer {} but it doesn't exist".format(
self.utc_partner_id, self.pk
)
)
else:
self.partner = None
self.university = None
super().save(*args, **kwargs)
class OfferSerializer(BaseModelSerializer):
class Meta:
model = Offer
fields = "__all__"
fields = BaseModelSerializer.Meta.fields + (
"year",
"semester",
"commend",
"double_degree",
"is_master_offered",
"specialties",
"partner",
"university",
)
class OfferViewSet(BaseModelViewSet):
......
from django.db import models
from backend_app.models.abstract.base import BaseModel
from backend_app.models.university import University
class Partner(BaseModel):
"""
Model that stores raw information from the UTC database regarding university partners.
Some universities in UTC database are present multiple times. To accommodate for this,
We store the data separetly and one REX-DRI university may link to multiple UTC partners.