Commit 9fa79975 authored by Florent Chehab's avatar Florent Chehab

Finish(external data): cron, mapping, offers, etc.

* Setup cron to automatically update remote data
* Support partial (login based) utc data update
* Tweaked models to record untouched or unlincked situations
* take them into account in the front and the back,
* Auto creation of general feedback also
* Added info about unlinked utc partners in the front
* Added button to request update ent data
* Added university offer module to the front

Other:
* Tweaked loading scripts
* tweaked models
* Fixed SelectField

Closes #28
parent cfb80538
Pipeline #42715 passed with stages
in 3 minutes and 54 seconds
......@@ -28,6 +28,9 @@ check_back:
paths:
- documentation/generated/
expire_in: 1 hour
variables:
FIXER_API_TOKEN: 91ed43e97a55f9ed9a501cc005c15e9c
UTC_API_ENDPOINT: http://192.168.122.1:8083/api
tags:
- docker
......@@ -56,6 +59,8 @@ test_back:
POSTGRES_PASSWORD: postgres
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432 # We absolutely need this one since a gitlab runner will inject a similar variable; will cause tests to fail.
FIXER_API_TOKEN: 91ed43e97a55f9ed9a501cc005c15e9c
UTC_API_ENDPOINT: http://192.168.122.1:8083/api
services:
- postgres:10.5
before_script:
......
......@@ -17,14 +17,15 @@ from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.for_testing.moderation import ForTestingModeration
from backend_app.models.for_testing.versioning import ForTestingVersioning
from backend_app.models.offer import Offer
from backend_app.models.partner import Partner
from backend_app.models.pendingModeration import PendingModeration
from backend_app.models.recommendationList import RecommendationList
from backend_app.models.taggedItems import UniversityTaggedItem, CountryTaggedItem
from backend_app.models.university import University
from backend_app.models.universityDri import UniversityDri
from backend_app.models.universityInfo import UniversityInfo
from backend_app.models.universityScholarship import UniversityScholarship
from backend_app.models.universitySemestersDates import UniversitySemestersDates
from backend_app.models.taggedItems import UniversityTaggedItem, CountryTaggedItem
from backend_app.models.userData import UserData
from backend_app.models.version import Version
from base_app.models import SiteInformation
......@@ -45,6 +46,7 @@ ALL_MODELS = [
Exchange,
ExchangeFeedback,
RecommendationList,
Partner,
University,
UniversityDri,
UniversityInfo,
......
......@@ -92,40 +92,46 @@ class LoadUniversityEx(LoadGeneric):
utc_allow_login=True,
)
ef = ExchangeFeedback.objects.create(
university=EPFL,
ef = ExchangeFeedback.objects.update_or_create(
exchange=exchange1,
general_comment="Very good",
academical_level_appreciation=5,
foreign_student_welcome=5,
cultural_interest=5,
)
defaults=dict(
university=EPFL,
general_comment="Very good",
academical_level_appreciation=5,
foreign_student_welcome=5,
cultural_interest=5,
),
)[0]
self.add_info_and_save(ef, self.admin)
exchange2 = Exchange.objects.create(
utc_partner_id=EPFL_as_partner.pk,
exchange2 = Exchange.objects.update_or_create(
utc_id=2,
student=self.admin,
year=2018,
semester="a",
duration=1,
double_degree=False,
master_obtained=False,
student_major="GI",
student_minor="FDD",
student_option="No",
utc_allow_courses=False,
utc_allow_login=False,
)
defaults=dict(
utc_partner_id=EPFL_as_partner.pk,
student=self.admin,
year=2018,
semester="a",
duration=1,
double_degree=False,
master_obtained=False,
student_major="GI",
student_minor="FDD",
student_option="No",
utc_allow_courses=False,
utc_allow_login=False,
),
)[0]
ef = ExchangeFeedback.objects.create(
university=EPFL,
ef = ExchangeFeedback.objects.update_or_create(
exchange=exchange2,
general_comment="Very good trop bien",
academical_level_appreciation=4,
foreign_student_welcome=3,
cultural_interest=4,
)
defaults=dict(
university=EPFL,
general_comment="Very good trop bien",
academical_level_appreciation=4,
foreign_student_welcome=3,
cultural_interest=4,
),
)[0]
self.add_info_and_save(ef, self.admin)
course1 = Course.objects.update_or_create(
......
# Generated by Django 2.1.7 on 2019-06-29 17:41
import django.core.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("backend_app", "0003_auto_20190629_1122")]
operations = [
migrations.AddField(
model_name="course",
name="unlinked",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="coursefeedback",
name="untouched",
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name="exchange",
name="unlinked",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="exchangefeedback",
name="untouched",
field=models.BooleanField(default=True),
),
migrations.AlterField(
model_name="course",
name="code",
field=models.CharField(blank=True, max_length=10, null=True),
),
migrations.AlterField(
model_name="exchange",
name="semester",
field=models.CharField(
choices=[("A", "autumn"), ("P", "spring")],
default="A",
max_length=1,
null=True,
),
),
migrations.AlterField(
model_name="exchange",
name="student_major",
field=models.CharField(blank=True, max_length=20),
),
migrations.AlterField(
model_name="exchange",
name="student_minor",
field=models.CharField(blank=True, max_length=47, null=True),
),
migrations.AlterField(
model_name="exchange",
name="student_option",
field=models.CharField(blank=True, max_length=7, null=True),
),
migrations.AlterField(
model_name="exchangefeedback",
name="academical_level_appreciation",
field=models.IntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(-5),
django.core.validators.MaxValueValidator(5),
],
),
),
migrations.AlterField(
model_name="exchangefeedback",
name="cultural_interest",
field=models.IntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(-5),
django.core.validators.MaxValueValidator(5),
],
),
),
migrations.AlterField(
model_name="exchangefeedback",
name="foreign_student_welcome",
field=models.IntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(-5),
django.core.validators.MaxValueValidator(5),
],
),
),
migrations.AlterField(
model_name="offer",
name="semester",
field=models.CharField(
choices=[("A", "autumn"), ("P", "spring")],
default="A",
max_length=2,
null=True,
),
),
migrations.AlterField(
model_name="partner",
name="address1",
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AlterField(
model_name="partner",
name="address2",
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AlterUniqueTogether(
name="offer", unique_together={("utc_partner_id", "year", "semester")}
),
]
......@@ -12,7 +12,7 @@ class Course(BaseModel):
# Not using utc_id as Primary Key here for our model to be more resilient
utc_id = models.IntegerField(null=False, unique=True)
utc_exchange_id = models.IntegerField(null=False)
code = models.CharField(max_length=10, null=True)
code = models.CharField(max_length=10, null=True, blank=True)
title = models.CharField(default="", null=False, blank=True, max_length=200)
link = models.URLField(null=True, blank=True, max_length=500)
ects = models.DecimalField(default=0, null=True, decimal_places=2, max_digits=7)
......@@ -21,6 +21,9 @@ class Course(BaseModel):
tsh_profile = models.CharField(null=True, blank=True, max_length=21)
student_login = models.CharField(null=True, blank=True, max_length=8)
# Field to tell that for some reason there is no corresponding exchange in the UTC DB
unlinked = models.BooleanField(default=False, null=False)
# a bit of denormalization
exchange = models.ForeignKey(
Exchange, on_delete=models.CASCADE, related_name="exchange_courses", null=True
......@@ -34,7 +37,7 @@ class Course(BaseModel):
exchange = Exchange.objects.get(utc_id=self.utc_exchange_id)
self.exchange = exchange
except Exchange.DoesNotExist:
logger.warning(
logger.error(
"Trying to find exchange {} "
"when updating course {} but it doesn't exist".format(
self.utc_exchange_id, self.utc_id
......
......@@ -29,3 +29,6 @@ class CourseFeedback(EssentialModule):
following_ease = models.IntegerField(
default=0, validators=[MinValueValidator(-5), MaxValueValidator(5)]
)
# Field to tell that the instance hasn't been edited as of now
untouched = models.BooleanField(default=True, null=False)
......@@ -18,14 +18,14 @@ class Exchange(BaseModel):
utc_partner_id = models.IntegerField(null=True)
year = models.PositiveIntegerField(default=2018, null=True)
semester = models.CharField(
max_length=5, choices=SEMESTER_OPTIONS, default="a", null=True
max_length=1, choices=SEMESTER_OPTIONS, default="A", null=True
)
duration = models.PositiveIntegerField(null=False)
double_degree = models.BooleanField(null=False)
master_obtained = models.BooleanField(null=False)
student_major = models.CharField(max_length=20, null=False)
student_minor = models.CharField(max_length=47, null=True)
student_option = models.CharField(max_length=7, null=True)
student_major = models.CharField(max_length=20, null=False, blank=True)
student_minor = models.CharField(max_length=47, null=True, blank=True)
student_option = models.CharField(max_length=7, null=True, blank=True)
utc_allow_courses = models.BooleanField(null=False)
utc_allow_login = models.BooleanField(null=False)
......@@ -36,10 +36,16 @@ class Exchange(BaseModel):
# (managned by signals on course save)
student = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
# Field to tell that for some reason there is no corresponding exchange in the UTC DB
unlinked = models.BooleanField(default=False, null=False)
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
Custom handling of denormalization and character
"""
if self.semester is not None:
self.semester = self.semester.upper()
if self.utc_partner_id is not None:
try:
self.partner = Partner.objects.get(utc_id=self.utc_partner_id)
......@@ -47,7 +53,7 @@ class Exchange(BaseModel):
except Partner.DoesNotExist:
self.partner = None
self.university = None
logger.warning(
logger.error(
"Trying to find partner {}"
"when updating exchange {} but it doesn't exist".format(
self.utc_partner_id, self.utc_id
......
......@@ -26,22 +26,29 @@ class ExchangeFeedback(EssentialModule):
)
general_comment = models.TextField(null=True, max_length=1500)
academical_level_appreciation = models.IntegerField(
validators=[MinValueValidator(-5), MaxValueValidator(5)]
default=0, validators=[MinValueValidator(-5), MaxValueValidator(5)]
)
foreign_student_welcome = models.IntegerField(
validators=[MinValueValidator(-5), MaxValueValidator(5)]
default=0, validators=[MinValueValidator(-5), MaxValueValidator(5)]
)
cultural_interest = models.IntegerField(
validators=[MinValueValidator(-5), MaxValueValidator(5)]
default=0, validators=[MinValueValidator(-5), MaxValueValidator(5)]
)
# A bit of denormalization (managed by signals)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
# Field to tell that the instance hasn't been edited as of now
untouched = models.BooleanField(default=True, null=False)
class ExchangeFeedbackSerializer(EssentialModuleSerializer):
exchange = ExchangeSerializer(read_only=True)
def update(self, instance, validated_data):
instance.untouched = False
return super().update(instance, validated_data)
class Meta:
model = ExchangeFeedback
fields = EssentialModuleSerializer.Meta.fields + (
......@@ -51,8 +58,9 @@ class ExchangeFeedbackSerializer(EssentialModuleSerializer):
"academical_level_appreciation",
"foreign_student_welcome",
"cultural_interest",
"untouched",
)
read_only_fields = ("university", "exchange")
read_only_fields = ("university", "exchange", "untouched")
class ExchangePermission(BasePermission):
......@@ -69,7 +77,10 @@ class ExchangeFeedbackViewSet(EssentialModuleViewSet):
NoDelete & NoPost & (ReadOnly | IsStaff | ExchangePermission),
)
queryset = (
ExchangeFeedback.objects.all()
ExchangeFeedback.objects.filter(
exchange__unlinked=False
) # only display linked instances
.order_by("-exchange__year", "exchange__semester")
.select_related("exchange", "updated_by", "moderated_by", "exchange__student")
.prefetch_related(
"exchange__exchange_courses",
......
......@@ -19,7 +19,7 @@ class Offer(BaseModel):
utc_partner_id = models.IntegerField(null=False)
year = models.PositiveIntegerField(default=2018, null=True)
semester = models.CharField(
max_length=2, choices=SEMESTER_OPTIONS, default="a", null=True
max_length=2, choices=SEMESTER_OPTIONS, default="A", null=True
)
comment = models.TextField(max_length=500, null=True)
......@@ -37,8 +37,13 @@ class Offer(BaseModel):
def save(self, *args, **kwargs):
"""
Custom handling of denormalization
Custom handling of denormalization and force character
"""
# make sure the semester is upper case
if self.semester is not None:
self.semester = self.semester.upper()
if self.utc_partner_id is not None:
try:
self.partner = Partner.objects.get(utc_id=self.utc_partner_id)
......@@ -46,7 +51,7 @@ class Offer(BaseModel):
except Partner.DoesNotExist:
self.partner = None
self.university = None
logger.warning(
logger.error(
"Trying to find partner {}"
"when updating offer {} but it doesn't exist".format(
self.utc_partner_id, self.pk
......@@ -57,6 +62,9 @@ class Offer(BaseModel):
self.university = None
super().save(*args, **kwargs)
class Meta:
unique_together = ("utc_partner_id", "year", "semester")
class OfferSerializer(BaseModelSerializer):
class Meta:
......@@ -64,17 +72,21 @@ class OfferSerializer(BaseModelSerializer):
fields = BaseModelSerializer.Meta.fields + (
"year",
"semester",
"commend",
"comment",
"double_degree",
"is_master_offered",
"specialties",
"partner",
"university",
"nb_seats_offered",
)
class OfferViewSet(BaseModelViewSet):
queryset = Offer.objects.all() # pylint: disable=E1101
queryset = Offer.objects.all().order_by(
"-year", "semester"
) # pylint: disable=E1101
serializer_class = OfferSerializer
permission_classes = (ReadOnly,)
end_point_route = "offers"
required_filterset_fields = ("university",)
filterset_fields = ("university",)
......@@ -15,8 +15,8 @@ class Partner(BaseModel):
# fields mapping directly to those of UTC db
utc_id = models.IntegerField(primary_key=True)
univ_name = models.CharField(max_length=80, null=False)
address1 = models.CharField(max_length=100, null=True)
address2 = models.CharField(max_length=100, null=True)
address1 = models.CharField(max_length=100, null=True, blank=True)
address2 = models.CharField(max_length=100, null=True, blank=True)
zipcode = models.CharField(max_length=40, null=True)
city = models.CharField(max_length=40, null=False)
country = models.CharField(max_length=50, null=False)
......@@ -41,7 +41,11 @@ class Partner(BaseModel):
if (
(previous.university is None and self.university is not None)
or (previous.university is not None and self.university is None)
or (previous.university.pk != self.university.pk)
or (
previous.university is not None
and self.university is not None
and previous.university.pk != self.university.pk
)
):
needs_to_propagate = True # university has change for the partner
except Partner.DoesNotExist:
......@@ -55,5 +59,5 @@ class Partner(BaseModel):
for exchange in Exchange.objects.filter(utc_partner_id=self.utc_id):
exchange.save() # Trigger denormalisation update
for offer in Offer.objects.filter(utc_partner_id=self.id):
for offer in Offer.objects.filter(utc_partner_id=self.utc_id):
offer.save() # Trigger denormalisation update
# This file is not a model. It is file to hold shared things across models.
SEMESTER_OPTIONS = (("a", "autumn"), ("p", "spring"))
SEMESTER_OPTIONS = (("A", "autumn"), ("P", "spring"))
......@@ -13,6 +13,10 @@ class CourseFeedbackSerializer(EssentialModuleSerializer):
def get_course_code(self, obj):
return obj.course.code
def update(self, instance, validated_data):
instance.untouched = False
return super().update(instance, validated_data)
class Meta:
model = CourseFeedback
fields = EssentialModuleSerializer.Meta.fields + (
......@@ -23,6 +27,7 @@ class CourseFeedbackSerializer(EssentialModuleSerializer):
"would_recommend",
"working_dose",
"following_ease",
"untouched",
)
......
......@@ -3,6 +3,8 @@ from django.db.models.signals import post_save
from backend_app.models.country import Country
from backend_app.models.course import Course
from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.exchange import Exchange
from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.taggedItems import (
UNIVERSITY_TAG_CHOICES,
UniversityTaggedItem,
......@@ -33,9 +35,16 @@ def create_user_modules(sender, instance, created, **kwargs):
UserData.objects.create(owner=instance)
def create_course_feedback_modules(sender, instance, created, **kwargs):
def create_exchange_feedback_module(sender, instance: Exchange, created, **kwargs):
if created:
CourseFeedback.objects.create(course=instance)
with revision_bot():
ExchangeFeedback.objects.create(exchange=instance)
def create_course_feedback_module(sender, instance, created, **kwargs):
if created:
with revision_bot():
CourseFeedback.objects.create(course=instance)
def create_tagged_items_university(sender, instance, created, **kwargs):
......@@ -61,6 +70,7 @@ def create_tagged_items_country(sender, instance, created, **kwargs):
def enable_auto_create():
post_save.connect(create_univ_modules, sender=University)
post_save.connect(create_user_modules, sender=User)
post_save.connect(create_course_feedback_modules, sender=Course)
post_save.connect(create_course_feedback_module, sender=Course)
post_save.connect(create_exchange_feedback_module, sender=Exchange)
post_save.connect(create_tagged_items_university, sender=University)
post_save.connect(create_tagged_items_country, sender=Country)
......@@ -19,6 +19,7 @@ def update_exchange_on_course_save(sender, instance: Course, created, **kwargs):
exchange.student = None
else:
exchange.student = User.objects.get(username=student_login)
exchange.save()
except User.DoesNotExist:
pass
......
......@@ -13,10 +13,6 @@ from backend_app.models.city import CityViewSet
from backend_app.models.country import CountryViewSet
from backend_app.models.countryDri import CountryDriViewSet
from backend_app.models.countryScholarship import CountryScholarshipViewSet
from backend_app.models.taggedItems import (
CountryTaggedItemViewSet,
UniversityTaggedItemViewSet,
)
from backend_app.models.course import Course
from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import CurrencyViewSet
......@@ -27,6 +23,7 @@ from backend_app.models.for_testing.moderation import ForTestingModerationViewSe
from backend_app.models.for_testing.versioning import ForTestingVersioningViewSet
from backend_app.models.language import LanguageViewSet
from backend_app.models.offer import OfferViewSet
from backend_app.models.partner import Partner
from backend_app.models.pendingModeration import (
PendingModerationViewSet,
PendingModerationObjViewSet,
......@@ -35,6 +32,10 @@ from backend_app.models.recommendationList import (
RecommendationListViewSet,
RecommendationList,
)
from backend_app.models.taggedItems import (
CountryTaggedItemViewSet,
UniversityTaggedItemViewSet,
)
from backend_app.models.university import UniversityViewSet
from backend_app.models.universityDri import UniversityDriViewSet
from backend_app.models.universityInfo import UniversityInfoViewSet
......@@ -50,6 +51,7 @@ from backend_app.serializers import (
)
from backend_app.settings.defaults import OBJ_MODERATION_PERMISSIONS
from base_app.models import UserViewset, User, SiteInformationViewSet
from external_data.management.commands.utils import UtcData
from external_data.models import ExternalDataUpdateInfo
......@@ -83,9 +85,10 @@ class CourseFeedbackViewSet(EssentialModuleViewSet):
permission_classes = (
NoDelete & NoPost & (ReadOnly | IsStaff | CourseFeedbackPermission),
)
queryset = CourseFeedback.objects.all().select_related(
queryset = CourseFeedback.objects.filter(course__unlinked=False).select_related(
"course", "updated_by", "moderated_by"
)
serializer_class = CourseFeedbackSerializer
end_point_route = "courseFeedbacks"
filterset_fields = ("course__exchange",)
......@@ -95,7 +98,7 @@ class CourseFeedbackViewSet(EssentialModuleViewSet):
class ExchangeViewSet(BaseModelViewSet):
permission_classes = (ReadOnly,)
queryset = (
Exchange.objects.all()
Exchange.objects.filter(unlinked=False)
.select_related("student")
.prefetch_related("exchange_courses", "exchange_courses__course_feedback")
)
......@@ -188,6 +191,34 @@ class LatestUpdateExternalDataViewSet(ViewSet):
)
class UnlinkedUtcPartners(ViewSet):
"""
Viewset to fetch the latest list of utc partners that are not linked to a univeristy
"""
permission_classes = (ReadOnly,)
end_point_route = "unlinkedUtcPartners"
def list(self, request):
partners = Partner.objects.filter(university=None)
return Response(list(map(lambda partner: partner.univ_name, partners)))
class UpdateStudentExchangesViewSet(ViewSet):
"""
Viewset to be able to ban and un-ban users from the site
"""
end_point_route = "updateStudentExchanges"
permission_classes = (ReadOnly,)
def list(self, request, **kwargs):
user = request.user
UtcData().update_one_student(user.username)
return Response()
class LogFrontendErrorsViewSet(ViewSet):
"""
Viewset to handle the logging of errors coming from the frontend.
......@@ -311,6 +342,8 @@ class RecommendationListChangeFollowerViewSet(ViewSet):
ALL_API_VIEW_VIEWSETS = [
AppModerationStatusViewSet,
LatestUpdateExternalDataViewSet,
UnlinkedUtcPartners,
UpdateStudentExchangesViewSet,
LogFrontendErrorsViewSet,
BannedUserViewSet,
RecommendationListChangeFollowerViewSet,
......
......@@ -18,7 +18,13 @@ class Command(BaseCommand):
help = "Command to handle user accounts emptying"
def handle(self, *args, **options):
logger.info("here")
ClearUserAccounts.run()
ClearSessions.run()
class ClearUserAccounts(object):
@staticmethod
def run():
for user in User.objects.filter(delete_next_time=True):
logger.info("Emptying account of user {}".format(user.pk))
......@@ -47,7 +53,10 @@ class Command(BaseCommand):
user.save()
# Finally we clear the sessions
class ClearSessions(object):
@staticmethod
def run():
ClearSessionCommand().handle()
ProxyGrantingTicket.clean_deleted_sessions()
SessionTicket.clean_deleted_sessions()
......
......@@ -293,7 +293,7 @@ LOGGING = {
"class": "django.utils.log.AdminEmailHandler",
},
"log_to_file_django": {
"level": "ERROR",
"level": "WARNING",
"filters": ["require_debug_false"],
"class": "logging.FileHandler",
"formatter": "verbose",
......
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base_app.settings.main")
import django # noqa: E402
django.setup()
# END OF SETUP
# DON'T PUT DJANGO RELATED IMPORTS BEFORE THIS