Commit 80323181 authored by Florent Chehab's avatar Florent Chehab
Browse files

Feat(pagination) & Feat(filter previous exchanges)

Pagination:
* Paginated endpoints for offers and previous exchanges,
* Added front component to display paginated data,
* Display offer in the front

Filter on previous exchanges:
* Added a denormalized model to store information
* Compute it in cron and in transaction
* Filter in front

Closes #123
parent 9fa79975
Pipeline #42718 passed with stages
in 3 minutes and 53 seconds
...@@ -12,7 +12,7 @@ from backend_app.models.countryScholarship import CountryScholarship ...@@ -12,7 +12,7 @@ from backend_app.models.countryScholarship import CountryScholarship
from backend_app.models.course import Course from backend_app.models.course import Course
from backend_app.models.courseFeedback import CourseFeedback from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import Currency from backend_app.models.currency import Currency
from backend_app.models.exchange import Exchange from backend_app.models.exchange import Exchange, UnivMajorMinors
from backend_app.models.exchangeFeedback import ExchangeFeedback from backend_app.models.exchangeFeedback import ExchangeFeedback
from backend_app.models.for_testing.moderation import ForTestingModeration from backend_app.models.for_testing.moderation import ForTestingModeration
from backend_app.models.for_testing.versioning import ForTestingVersioning from backend_app.models.for_testing.versioning import ForTestingVersioning
...@@ -45,6 +45,7 @@ ALL_MODELS = [ ...@@ -45,6 +45,7 @@ ALL_MODELS = [
PendingModeration, PendingModeration,
Exchange, Exchange,
ExchangeFeedback, ExchangeFeedback,
UnivMajorMinors,
RecommendationList, RecommendationList,
Partner, Partner,
University, University,
......
...@@ -85,7 +85,7 @@ class LoadUniversityEx(LoadGeneric): ...@@ -85,7 +85,7 @@ class LoadUniversityEx(LoadGeneric):
duration=1, duration=1,
double_degree=False, double_degree=False,
master_obtained=False, master_obtained=False,
student_major="GI", student_major_and_semester="GI6",
student_minor="FDD", student_minor="FDD",
student_option="No", student_option="No",
utc_allow_courses=True, utc_allow_courses=True,
...@@ -114,7 +114,7 @@ class LoadUniversityEx(LoadGeneric): ...@@ -114,7 +114,7 @@ class LoadUniversityEx(LoadGeneric):
duration=1, duration=1,
double_degree=False, double_degree=False,
master_obtained=False, master_obtained=False,
student_major="GI", student_major_and_semester="GI4",
student_minor="FDD", student_minor="FDD",
student_option="No", student_option="No",
utc_allow_courses=False, utc_allow_courses=False,
......
# Generated by Django 2.1.7 on 2019-06-30 07:37
import backend_app.fields
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [("backend_app", "0004_auto_20190629_1941")]
operations = [
migrations.CreateModel(
name="UnivMajorMinors",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("major", models.CharField(blank=True, max_length=20, null=True)),
("minors", backend_app.fields.JSONField(default=list)),
(
"university",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="backend_app.University",
),
),
],
),
migrations.AddField(
model_name="exchange",
name="student_major_and_semester",
field=models.CharField(blank=True, max_length=20),
),
migrations.AddField(
model_name="exchange",
name="student_semester",
field=models.IntegerField(null=True),
),
migrations.AlterField(
model_name="exchange",
name="student_major",
field=models.CharField(blank=True, max_length=20, null=True),
),
migrations.AlterUniqueTogether(
name="univmajorminors", unique_together={("university", "major")}
),
]
import logging import logging
import re
from django.db import models from django.db import models, transaction
from backend_app.models.abstract.base import BaseModel from backend_app.fields import JSONField
from backend_app.models.abstract.base import (
BaseModel,
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.models.partner import Partner from backend_app.models.partner import Partner
from backend_app.models.shared import SEMESTER_OPTIONS from backend_app.models.shared import SEMESTER_OPTIONS
from backend_app.models.university import University from backend_app.models.university import University
from backend_app.permissions.app_permissions import ReadOnly
from base_app.models import User from base_app.models import User
logger = logging.getLogger("django") logger = logging.getLogger("django")
...@@ -23,7 +30,7 @@ class Exchange(BaseModel): ...@@ -23,7 +30,7 @@ class Exchange(BaseModel):
duration = models.PositiveIntegerField(null=False) duration = models.PositiveIntegerField(null=False)
double_degree = models.BooleanField(null=False) double_degree = models.BooleanField(null=False)
master_obtained = models.BooleanField(null=False) master_obtained = models.BooleanField(null=False)
student_major = models.CharField(max_length=20, null=False, blank=True) student_major_and_semester = models.CharField(max_length=20, null=False, blank=True)
student_minor = models.CharField(max_length=47, null=True, blank=True) student_minor = models.CharField(max_length=47, null=True, blank=True)
student_option = models.CharField(max_length=7, null=True, blank=True) student_option = models.CharField(max_length=7, null=True, blank=True)
...@@ -31,6 +38,8 @@ class Exchange(BaseModel): ...@@ -31,6 +38,8 @@ class Exchange(BaseModel):
utc_allow_login = models.BooleanField(null=False) utc_allow_login = models.BooleanField(null=False)
# a bit of denormalization # a bit of denormalization
student_major = models.CharField(max_length=20, null=True, blank=True)
student_semester = models.IntegerField(null=True)
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=True) partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=True)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True) university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
# (managned by signals on course save) # (managned by signals on course save)
...@@ -63,4 +72,78 @@ class Exchange(BaseModel): ...@@ -63,4 +72,78 @@ class Exchange(BaseModel):
self.partner = None self.partner = None
self.university = None self.university = None
# Updating student major and semester
regex = r"^(\w+)(\d+)$"
search = re.search(regex, self.student_major_and_semester)
if search is not None:
self.student_major = search.group(1)
self.student_semester = search.group(2)
else:
self.student_semester = None
self.student_minor = None
super().save(*args, **kwargs) super().save(*args, **kwargs)
#####
#####
#####
#####
#####
#####
class UnivMajorMinors(BaseModel):
"""
Model to store denormalize data about all the exchanges
"""
university = models.ForeignKey(University, on_delete=models.CASCADE, null=False)
major = models.CharField(max_length=20, null=True, blank=True)
minors = JSONField(default=list)
class Meta:
unique_together = ("university", "major")
class UnivMajorMinorsSerializer(BaseModelSerializer):
class Meta:
model = UnivMajorMinors
fields = BaseModelSerializer.Meta.fields + ("university", "major", "minors")
class UnivMajorMinorsViewSet(BaseModelViewSet):
queryset = UnivMajorMinors.objects.all() # pylint: disable=E1101
serializer_class = UnivMajorMinorsSerializer
permission_classes = (ReadOnly,)
end_point_route = "univMajorMinors"
filterset_fields = ("university",)
required_filterset_fields = ("university",)
@transaction.atomic
def update_denormalized_univ_major_minor():
logger.info("Computing the denormalized univ, major and minor")
data = {}
for exchange in Exchange.objects.all():
university = exchange.university
if university is None:
continue
student_major = exchange.student_major
student_minor = exchange.student_minor
if university not in data.keys():
data[university] = {}
if student_major not in data[university].keys():
data[university][student_major] = set()
data[university][student_major].add(student_minor)
UnivMajorMinors.objects.all().delete()
for university, majors_and_minors in data.items():
for major, minors in majors_and_minors.items():
UnivMajorMinors.objects.update_or_create(
university=university, major=major, defaults=dict(minors=list(minors))
)
logger.info("Done computing the denormarlize major and minor")
...@@ -12,6 +12,7 @@ from backend_app.models.university import University ...@@ -12,6 +12,7 @@ from backend_app.models.university import University
from backend_app.permissions.app_permissions import ReadOnly, IsStaff, NoDelete, NoPost from backend_app.permissions.app_permissions import ReadOnly, IsStaff, NoDelete, NoPost
from backend_app.permissions.moderation import ModerationLevels from backend_app.permissions.moderation import ModerationLevels
from backend_app.serializers import ExchangeSerializer from backend_app.serializers import ExchangeSerializer
from backend_app.utils import CustomPagination
class ExchangeFeedback(EssentialModule): class ExchangeFeedback(EssentialModule):
...@@ -91,5 +92,10 @@ class ExchangeFeedbackViewSet(EssentialModuleViewSet): ...@@ -91,5 +92,10 @@ class ExchangeFeedbackViewSet(EssentialModuleViewSet):
) )
serializer_class = ExchangeFeedbackSerializer serializer_class = ExchangeFeedbackSerializer
end_point_route = "exchangeFeedbacks" end_point_route = "exchangeFeedbacks"
filterset_fields = ("university",) filterset_fields = (
"university",
"exchange__student_major",
"exchange__student_minor",
)
required_filterset_fields = ("university",) required_filterset_fields = ("university",)
pagination_class = CustomPagination
...@@ -11,6 +11,7 @@ from backend_app.models.partner import Partner ...@@ -11,6 +11,7 @@ from backend_app.models.partner import Partner
from backend_app.models.shared import SEMESTER_OPTIONS from backend_app.models.shared import SEMESTER_OPTIONS
from backend_app.models.university import University from backend_app.models.university import University
from backend_app.permissions.app_permissions import ReadOnly from backend_app.permissions.app_permissions import ReadOnly
from backend_app.utils import CustomPagination
logger = logging.getLogger("django") logger = logging.getLogger("django")
...@@ -90,3 +91,4 @@ class OfferViewSet(BaseModelViewSet): ...@@ -90,3 +91,4 @@ class OfferViewSet(BaseModelViewSet):
end_point_route = "offers" end_point_route = "offers"
required_filterset_fields = ("university",) required_filterset_fields = ("university",)
filterset_fields = ("university",) filterset_fields = ("university",)
pagination_class = CustomPagination
...@@ -60,6 +60,7 @@ EXCHANGE_FIELDS = BaseModelSerializer.Meta.fields + ( ...@@ -60,6 +60,7 @@ EXCHANGE_FIELDS = BaseModelSerializer.Meta.fields + (
"duration", "duration",
"double_degree", "double_degree",
"master_obtained", "master_obtained",
"student_major_and_semester",
"student_major", "student_major",
"student_minor", "student_minor",
"student_option", "student_option",
......
...@@ -7,6 +7,8 @@ from typing import Dict ...@@ -7,6 +7,8 @@ from typing import Dict
import reversion import reversion
from django.conf import settings from django.conf import settings
from django.utils import timezone from django.utils import timezone
from rest_framework.pagination import PageNumberPagination
from rest_framework.response import Response
from backend_app.settings.defaults import OBJ_MODERATION_PERMISSIONS from backend_app.settings.defaults import OBJ_MODERATION_PERMISSIONS
from base_app.settings.dir_locations import REPO_ROOT_DIR from base_app.settings.dir_locations import REPO_ROOT_DIR
...@@ -56,6 +58,31 @@ def get_default_theme_settings(): ...@@ -56,6 +58,31 @@ def get_default_theme_settings():
return json.load(f) return json.load(f)
class CustomPagination(PageNumberPagination):
page_size = 10
page_size_query_param = "page_size"
max_page_size = 500
def get_paginated_response(self, data):
previous_link = self.get_previous_link()
next_link = self.get_next_link()
return Response(
dict(
first=previous_link is None,
last=next_link is None,
page=self.page.number,
pages_count=self.page.paginator.num_pages,
links=dict(
next=self.get_next_link(), previous=self.get_previous_link()
),
number_elements=self.page.paginator.count,
page_size=self.page_size,
content=data,
)
)
__BOT_USER_CACHE = None __BOT_USER_CACHE = None
......
...@@ -16,7 +16,7 @@ from backend_app.models.countryScholarship import CountryScholarshipViewSet ...@@ -16,7 +16,7 @@ from backend_app.models.countryScholarship import CountryScholarshipViewSet
from backend_app.models.course import Course from backend_app.models.course import Course
from backend_app.models.courseFeedback import CourseFeedback from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import CurrencyViewSet from backend_app.models.currency import CurrencyViewSet
from backend_app.models.exchange import Exchange from backend_app.models.exchange import Exchange, UnivMajorMinorsViewSet
from backend_app.models.exchangeFeedback import ExchangeFeedbackViewSet from backend_app.models.exchangeFeedback import ExchangeFeedbackViewSet
from backend_app.models.file_picture import FileViewSet, PictureViewSet from backend_app.models.file_picture import FileViewSet, PictureViewSet
from backend_app.models.for_testing.moderation import ForTestingModerationViewSet from backend_app.models.for_testing.moderation import ForTestingModerationViewSet
...@@ -129,6 +129,7 @@ ALL_API_VIEWSETS = [ ...@@ -129,6 +129,7 @@ ALL_API_VIEWSETS = [
PictureViewSet, PictureViewSet,
ExchangeViewSet, ExchangeViewSet,
ExchangeFeedbackViewSet, ExchangeFeedbackViewSet,
UnivMajorMinorsViewSet,
RecommendationListViewSet, RecommendationListViewSet,
UniversityViewSet, UniversityViewSet,
UniversityDriViewSet, UniversityDriViewSet,
......
...@@ -10,7 +10,10 @@ django.setup() ...@@ -10,7 +10,10 @@ django.setup()
import logging # noqa: E402 import logging # noqa: E402
from uwsgidecorators import harakiri, cron # noqa: E402 from uwsgidecorators import harakiri, cron, timer # noqa: E402
from backend_app.models.exchange import (
update_denormalized_univ_major_minor,
) # noqa: E402
from external_data.management.commands.utils import FixerData, UtcData # noqa: E402 from external_data.management.commands.utils import FixerData, UtcData # noqa: E402
from base_app.management.commands.clean_user_accounts import ( from base_app.management.commands.clean_user_accounts import (
...@@ -34,6 +37,12 @@ def update_utc_ent(num): ...@@ -34,6 +37,12 @@ def update_utc_ent(num):
UtcData().update() UtcData().update()
@timer(60 * 60, target="spooler") # run it every hour
@harakiri(60)
def update_univ_major_minor():
update_denormalized_univ_major_minor()
@cron(20, 0, -1, -1, -1, target="spooler") # everyday at 20 past midnight @cron(20, 0, -1, -1, -1, target="spooler") # everyday at 20 past midnight
@harakiri(60 * 5) # shouldn't take more than 5 minutes to run @harakiri(60 * 5) # shouldn't take more than 5 minutes to run
def clear_and_clean_sessions(num): def clear_and_clean_sessions(num):
......
...@@ -5,7 +5,7 @@ import requests ...@@ -5,7 +5,7 @@ import requests
from backend_app.models.course import Course from backend_app.models.course import Course
from backend_app.models.currency import Currency from backend_app.models.currency import Currency
from backend_app.models.exchange import Exchange from backend_app.models.exchange import Exchange, update_denormalized_univ_major_minor
from backend_app.models.offer import Offer from backend_app.models.offer import Offer
from backend_app.models.partner import Partner from backend_app.models.partner import Partner
from external_data.models import ExternalDataUpdateInfo from external_data.models import ExternalDataUpdateInfo
...@@ -90,6 +90,8 @@ class UtcData(object): ...@@ -90,6 +90,8 @@ class UtcData(object):
self.__update_invalidated() self.__update_invalidated()
logger.info("Updating UTC info done !") logger.info("Updating UTC info done !")
update_denormalized_univ_major_minor()
def __update_invalidated(self): def __update_invalidated(self):
""" """
Function to update the unlinked status of exchanges and courses. Function to update the unlinked status of exchanges and courses.
...@@ -183,7 +185,7 @@ class UtcData(object): ...@@ -183,7 +185,7 @@ class UtcData(object):
duration=exchange["duree"], duration=exchange["duree"],
double_degree=exchange["doubleDiplome"], double_degree=exchange["doubleDiplome"],
master_obtained=exchange["master"], master_obtained=exchange["master"],
student_major=exchange["specialite"], student_major_and_semester=exchange["specialite"],
student_minor=exchange["option"], student_minor=exchange["option"],
utc_allow_courses=exchange["autorisationTransfertUv"], utc_allow_courses=exchange["autorisationTransfertUv"],
utc_allow_login=exchange["autorisationTransfertLogin"], utc_allow_login=exchange["autorisationTransfertLogin"],
......
import React from "react";
import {makeStyles} from "@material-ui/core/styles";
import MobileStepper from "@material-ui/core/MobileStepper";
import Button from "@material-ui/core/Button";
import KeyboardArrowLeft from "@material-ui/icons/KeyboardArrowLeft";
import KeyboardArrowRight from "@material-ui/icons/KeyboardArrowRight";
import PropTypes from "prop-types";
const useStyles = makeStyles(theme => ({
root: {
maxWidth: 800,
margin: "0 auto"
},
stepperContainer: {
width: "100%",
},
progress: {
[theme.breakpoints.up("sm")]: {
width: "65%"
},
[theme.breakpoints.down("sm")]: {
width: "60%"
},
[theme.breakpoints.down("xs")]: {
width: "30%"
},
}
}));
function PaginatedData(props) {
const classes = useStyles(),
{data, goToPage, render, stepperOnBottom, stepperOnTop, EmptyMessageComponent} = props,
{page: activeStep, last, first, pages_count: nbOfPages, content, number_elements: totalElements} = data;
if (totalElements === 0){
return EmptyMessageComponent;
}
function renderStepper() {
if (nbOfPages === 1) {
return <></>;
}
return (
<div className={classes.stepperContainer}>
<MobileStepper
variant="progress"
steps={nbOfPages}
position="static"
activeStep={activeStep - 1}
className={classes.root}
classes={{progress: classes.progress}}
backButton={
<Button size="small" onClick={() => goToPage(activeStep - 1)} disabled={first}>
<KeyboardArrowLeft/>
Précédent
</Button>
}
nextButton={
<Button size="small" onClick={() => goToPage(activeStep + 1)} disabled={last}>
Suivant
<KeyboardArrowRight/>
</Button>
}
/>
</div>
);
}
return (
<div>
{stepperOnTop ? renderStepper() : <></>}
<div>
{
content.map(dataEl => render(dataEl))
}
</div>
{stepperOnBottom ? renderStepper() : <></>}
</div>
);
}
PaginatedData.defaultProps = {
stepperOnTop: false,
stepperOnBottom: false,
};
PaginatedData.propTypes = {
data: PropTypes.shape({
first: PropTypes.bool.isRequired,
last: PropTypes.bool.isRequired,
page: PropTypes.number.isRequired,
pages_count: PropTypes.number.isRequired,
number_elements: PropTypes.number.isRequired,
page_size: PropTypes.number.isRequired,
content: PropTypes.array.isRequired,
}).isRequired,
goToPage: PropTypes.func.isRequired,
render: PropTypes.func.isRequired,
stepperOnTop: PropTypes.bool,
stepperOnBottom: PropTypes.bool,
EmptyMessageComponent: PropTypes.node.isRequired,
};
export default PaginatedData;
\ No newline at end of file
...@@ -11,11 +11,12 @@ import {withUnivInfo} from "../common/withUnivInfo"; ...@@ -11,11 +11,12 @@ import {withUnivInfo} from "../common/withUnivInfo";
import {RequestParams} from "../../../redux/api/RequestParams"; import {RequestParams} from "../../../redux/api/RequestParams";
import CustomComponentForAPI from "../../common/CustomComponentForAPI"; import CustomComponentForAPI from "../../common/CustomComponentForAPI";
import Paper from "@material-ui/core/Paper"; import Paper from "@material-ui/core/Paper";
import {makeStyles} from "@material-ui/styles"; import {makeStyles, withStyles} from "@material-ui/styles";
import Typography from "@material-ui/core/Typography"; import Typography from "@material-ui/core/Typography";
import Chip from "@material-ui/core/Chip"; import Chip from "@material-ui/core/Chip";
import PaginatedData from "../../common/PaginatedData";
const useStyle = makeStyles(theme => ({ const style = theme => ({
paper: { paper: {