Commit 638abfbf authored by Florent Chehab's avatar Florent Chehab
Browse files

feat(more complex filters)

* Added denormalized data about semesters, majors and minors in university model
* added function to compute it
* Added function to cron
* new FilterHandler in the front to handle / cache the filtering
* Tweaked map and search components to display the right stuff
* Map now displays in a different color the elements that have been filtered

* Renamed spacilities field to majors
* Fixed bugged in downshift multiple (couldn't add same after delete)
* Deleted useless code in offer

Linked to #31
parent 048144cc
Pipeline #42724 passed with stages
in 3 minutes and 54 seconds
# Generated by Django 2.1.7 on 2019-06-30 15:53
import backend_app.fields
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [("backend_app", "0006_auto_20190630_1119")]
operations = [
migrations.RenameField(
model_name="offer", old_name="specialties", new_name="majors"
),
migrations.AddField(
model_name="university",
name="denormalized_infos",
field=backend_app.fields.JSONField(default=dict),
),
migrations.AlterField(
model_name="exchange",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
related_name="exchanges",
to="backend_app.University",
),
),
migrations.AlterField(
model_name="offer",
name="university",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.PROTECT,
related_name="offers",
to="backend_app.University",
),
),
]
......@@ -41,7 +41,9 @@ class Exchange(BaseModel):
student_major = models.CharField(max_length=20, null=True, blank=True)
student_semester = models.IntegerField(null=True)
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=True)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
university = models.ForeignKey(
University, on_delete=models.PROTECT, null=True, related_name="exchanges"
)
# (managned by signals on course save)
student = models.ForeignKey(
User, on_delete=models.CASCADE, null=True, related_name="exchanges"
......@@ -127,7 +129,7 @@ class UnivMajorMinorsViewSet(BaseModelViewSet):
def update_denormalized_univ_major_minor():
logger.info("Computing the denormalized univ, major and minor")
data = {}
for exchange in Exchange.objects.all():
for exchange in Exchange.objects.all().prefetch_related("university"):
university = exchange.university
if university is None:
continue
......
......@@ -30,11 +30,13 @@ class Offer(BaseModel):
nb_seats_offered = models.PositiveIntegerField(null=True)
# null => exchange not possible
specialties = models.CharField(max_length=4000, null=True)
majors = models.CharField(max_length=4000, null=True)
# A bit of denormalization
partner = models.ForeignKey(Partner, on_delete=models.PROTECT, null=False)
university = models.ForeignKey(University, on_delete=models.PROTECT, null=True)
university = models.ForeignKey(
University, on_delete=models.PROTECT, null=True, related_name="offers"
)
def save(self, *args, **kwargs):
"""
......@@ -76,7 +78,7 @@ class OfferSerializer(BaseModelSerializer):
"comment",
"double_degree",
"is_master_offered",
"specialties",
"majors",
"university",
"nb_seats_offered",
)
......
import logging
from django.conf import settings
from django.db import models
from backend_app.fields import JSONField
from backend_app.models.abstract.essentialModule import (
EssentialModule,
EssentialModuleSerializer,
......@@ -8,6 +11,8 @@ from backend_app.models.abstract.essentialModule import (
)
from backend_app.validation.validators import PathExtensionValidator
logger = logging.getLogger("django")
class University(EssentialModule):
"""
......@@ -23,6 +28,9 @@ class University(EssentialModule):
)
website = models.URLField(default="", blank=True, max_length=300)
# a bit of denormalization
denormalized_infos = JSONField(default=dict)
class UniversitySerializer(EssentialModuleSerializer):
class Meta:
......@@ -32,6 +40,7 @@ class UniversitySerializer(EssentialModuleSerializer):
"acronym",
"logo",
"website",
"denormalized_infos",
)
......@@ -39,3 +48,44 @@ class UniversityViewSet(EssentialModuleViewSet):
serializer_class = UniversitySerializer
queryset = University.objects.all() # pylint: disable=E1101
end_point_route = "universities"
def update_denormalized_univ_field():
logger.info("Computing the denormalized offers/semester/major in university")
for university in University.objects.all().prefetch_related("offers", "exchanges"):
denormalized_infos = dict()
# handling of offers
for offer in university.offers.all():
semester = "{}{}".format(offer.semester, offer.year)
if semester not in denormalized_infos.keys():
denormalized_infos[semester] = dict()
majors = offer.majors
if majors is not None:
possibilities = set(
map(lambda s: s.lstrip().rstrip(), majors.split(","))
)
for major in possibilities:
# No minors in offers
denormalized_infos[semester][major] = list()
# handling of exchanges
for exchange in university.exchanges.all():
semester = "{}{}".format(exchange.semester, exchange.year)
if semester not in denormalized_infos.keys():
denormalized_infos[semester] = dict()
major = exchange.student_major
minor = exchange.student_minor
if major is not None:
if major not in denormalized_infos[semester].keys():
denormalized_infos[semester][major] = list()
if (
minor is not None
and minor not in denormalized_infos[semester][major]
):
denormalized_infos[semester][major].append(minor)
university.denormalized_infos = denormalized_infos
university.save()
logger.info("Done the denormalized offers/semester/major in university")
......@@ -15,6 +15,9 @@ from backend_app.models.exchange import (
update_denormalized_univ_major_minor,
) # noqa: E402
from backend_app.models.university import (
update_denormalized_univ_field,
) # noqa: E402
from external_data.management.commands.utils import FixerData, UtcData # noqa: E402
from base_app.management.commands.clean_user_accounts import (
ClearUserAccounts,
......@@ -39,8 +42,9 @@ def update_utc_ent(num):
@timer(60 * 60, target="spooler") # run it every hour
@harakiri(60)
def update_univ_major_minor():
def update_extra_denormalization():
update_denormalized_univ_major_minor()
update_denormalized_univ_field()
@cron(20, 0, -1, -1, -1, target="spooler") # everyday at 20 past midnight
......
......@@ -8,6 +8,7 @@ from backend_app.models.currency import Currency
from backend_app.models.exchange import Exchange, update_denormalized_univ_major_minor
from backend_app.models.offer import Offer
from backend_app.models.partner import Partner
from backend_app.models.university import update_denormalized_univ_field
from external_data.models import ExternalDataUpdateInfo
logger = logging.getLogger("django")
......@@ -91,6 +92,7 @@ class UtcData(object):
logger.info("Updating UTC info done !")
update_denormalized_univ_major_minor()
update_denormalized_univ_field()
def __update_invalidated(self):
"""
......@@ -212,7 +214,7 @@ class UtcData(object):
nb_seats_offered=destination["nombrePlaces"],
double_degree=destination["doubleDiplome"],
is_master_offered=destination["master"],
specialties=destination["branches"],
majors=destination["branches"],
),
)
......
......@@ -123,6 +123,8 @@ class DownshiftMultiple extends React.Component {
};
handleChange = val => {
if (val === null) return; // on reset will be null
let {value} = this.state;
if (value.indexOf(val) === -1) {
value = [...value, val];
......@@ -142,8 +144,7 @@ class DownshiftMultiple extends React.Component {
handleDelete = val => () => {
this.setState(state => {
const value = [...state.value];
value.splice(value.indexOf(val), 1);
const value = state.value.filter(v => val !== v);
this.onChange(value);
return {value};
});
......@@ -167,7 +168,7 @@ class DownshiftMultiple extends React.Component {
<Downshift inputValue={inputValue}
onChange={this.handleChange}>
{
({getInputProps, getItemProps, isOpen, inputValue: inputValue2, selectedItem: selectedItem2, highlightedIndex,}) =>
({getInputProps, getItemProps, isOpen, inputValue: inputValue2, selectedItem: selectedItem2, highlightedIndex, clearSelection}) =>
(
<div className={classes.container}>
{renderInput({
......@@ -180,7 +181,10 @@ class DownshiftMultiple extends React.Component {
tabIndex={-1}
label={this.reverseOptions.get(val)}
className={classes.chip}
onDelete={this.handleDelete(val)}
onDelete={() => {
this.handleDelete(val)();
clearSelection();
}}
variant="outlined"
color="primary"
/>
......@@ -225,7 +229,7 @@ DownshiftMultiple.propTypes = {
fieldPlaceholder: PropTypes.string.isRequired,
options: PropTypes.arrayOf(PropTypes.shape({
label: PropTypes.string.isRequired,
value: PropTypes.oneOfType([PropTypes.string, PropTypes.number]).isRequired,
value: PropTypes.oneOfType([PropTypes.string.isRequired, PropTypes.number.isRequired]).isRequired, // NEVER PUT A NULL
})).isRequired,
multiple: PropTypes.bool.isRequired, // do we allow multiple values to be selected.
cacheId: PropTypes.string, // if not null enable cachig of value and input Value
......
/* eslint-disable indent */
import React from "react";
import PropTypes from "prop-types";
import DownshiftMultiple from "../common/DownshiftMultiple";
......@@ -16,6 +17,217 @@ import InfoIcon from "@material-ui/icons/InfoOutlined";
import getActions from "../../redux/api/getActions";
import {compose} from "recompose";
import uuid from "uuid/v4";
import {UniversityHelper} from "../../redux/api/helpers";
/**
* Class that handle all the filter manipulation with caching
*/
class FilterHandler {
constructor(data) {
this.countries = data.countries;
this.cities = data.cities;
this.mainCampuses = data.mainCampuses;
this.universities = data.universities;
}
getUnivForUnivId(univId) {
return UniversityHelper.getUniv(univId);
}
static _universitiesCountriesCache = undefined;
get universityIdsCountries() {
if (typeof FilterHandler._universitiesCountriesCache === "undefined") {
const {mainCampuses, cities, countries} = this;
let citiesMap = new Map(),
countriesMap = new Map();
cities.forEach(el => citiesMap.set(el.id, el));
countries.forEach(el => countriesMap.set(el.id, el));
let res = new Map();
mainCampuses.forEach(el => {
const cityId = el.city,
city = citiesMap.get(cityId),
countryId = city.country,
country = countriesMap.get(countryId);
res.set(el.university, country);
});
FilterHandler._universitiesCountriesCache = res;
}
return FilterHandler._universitiesCountriesCache;
}
static _countriesWereThereAreUniversities = undefined;
/**
* Function to get the list of countries where there are universities.
*
* @returns {Array} of the countries instances
*/
get countriesWhereThereAreUniversities() {
if (typeof FilterHandler._countriesWereThereAreUniversities === "undefined") {
const out = new Map();
this.universityIdsCountries.forEach((country) => out.set(country.id, country));
FilterHandler._countriesWereThereAreUniversities = [...out.values()];
}
return FilterHandler._countriesWereThereAreUniversities;
}
static _countriesOptions = undefined;
get countriesOptions() {
if (typeof FilterHandler._countriesOptions === "undefined") {
FilterHandler._countriesOptions = this.countriesWhereThereAreUniversities
.map(c => ({value: c.iso_alpha2_code, label: c.name}));
}
return FilterHandler._countriesOptions;
}
static _majorMinorOptions = undefined;
/**
* @return {array.<object>}
*/
get majorMinorOptions() {
if (typeof FilterHandler._majorMinorOptions === "undefined") {
const out = new Set(
this.universities
.flatMap(u => this.getMajorMinorsInUniv(u))
);
FilterHandler._majorMinorOptions = [...out].map(el => ({value: el, label: el}));
}
return FilterHandler._majorMinorOptions;
}
getMajorInUniv(univObj) {
return [...new Set(
Object.values(univObj.denormalized_infos)
.flatMap(forSemester => Object.keys(forSemester))
)];
}
/**
* @param univObj
* @param allowedSemesters
* @return {array.<string>}
*/
getMajorMinorsInUniv(univObj, allowedSemesters = null) {
const realMinors = Object.entries(univObj.denormalized_infos)
.filter(([sem,]) =>
allowedSemesters === null ? true : allowedSemesters.includes(sem)
)
.map(([, forSemester]) => forSemester)
.flatMap(forSemester => Object.entries(forSemester))
.flatMap(([major, minors]) => minors.map(minor => `${major}${minor}`));
const extraMinors = this.getMajorInUniv(univObj).map(major => `${major} — Toutes filières confondues`);
return [...new Set(realMinors), ...extraMinors];
}
getSemestersInUniv(univObj) {
return Object.keys(univObj.denormalized_infos);
}
static _semesterOptions = undefined;
/**
* @return {array.<object>}
*/
get semesterOptions() {
if (typeof FilterHandler._semesterOptions === "undefined") {
const out = new Set(
this.universities
.flatMap(u => this.getSemestersInUniv(u))
);
FilterHandler._semesterOptions = [...out].map(semester => ({value: semester, label: semester}));
}
return FilterHandler._semesterOptions;
}
getCountryCodeForUniversity(univId) {
const id = parseInt(univId);
return this.universityIdsCountries.get(id).id;
}
/**
*
* @param {array.<string>} countryCodes
* @return {array.<object>}
*/
getUniversitiesInCountries(countryCodes) {
const possiblesCountries = new Set(countryCodes);
const out = [];
this.universityIdsCountries.forEach((country, univId) => {
if (possiblesCountries.has(country.id)) out.push(univId);
});
return out;
}
/**
*
* @param {array.<string>} selectedCountriesCode
* @param {array.<string>} selectedSemesters
* @param {array.<string>} selectedMajorMinors
* @return {array.<number>}
*/
getSelection(selectedCountriesCode = [], selectedSemesters = [], selectedMajorMinors = []) {
let possible = selectedCountriesCode.length === 0 ?
this.universities
:
this.getUniversitiesInCountries(selectedCountriesCode).map(id => this.getUnivForUnivId(id));
if (selectedSemesters.length > 0) {
const possibleSemesters = new Set(selectedSemesters);
possible = possible.filter(
univ => {
const semestersInUniv = this.getSemestersInUniv(univ);
return semestersInUniv.some(semester => possibleSemesters.has(semester));
}
);
}
if (selectedMajorMinors.length > 0) {
const possibleMajorMinors = new Set(selectedMajorMinors);
possible = possible.filter(
univ => {
const majorMinorsInUniv = new Set(this.getMajorMinorsInUniv(univ));
return [...majorMinorsInUniv].some(el => possibleMajorMinors.has(el));
}
);
}
if (selectedSemesters.length > 0 && selectedMajorMinors.length > 0) {
const possibleMajorMinors = new Set(selectedMajorMinors);
const possibleSemesters = new Set(selectedSemesters);
possible = possible.filter(
univ => {
const semestersInUniv = this.getSemestersInUniv(univ);
const possibleSemesterInUniv = semestersInUniv.filter(sem => possibleSemesters.has(sem));
return this.getMajorMinorsInUniv(univ, possibleSemesterInUniv)
.some(el => possibleMajorMinors.has(el));
}
);
}
return possible.map(univ => univ.id);
}
}
/**
......@@ -30,62 +242,66 @@ class Filter extends CustomComponentForAPI {
/**
* Static variables to share behaviors between instances
*/
static DOWNSHIFT_ID = uuid();
static DOWNSHIFT_COUNTRIES_ID = uuid();
static DOWNSHIFT_SEMESTERS_ID = uuid();
static DOWNSHIFT_MAJORS_ID = uuid();
static isOpened = false;
static hasSelection = false;
static nbSelection = 0;
/**
* Function to get the list of countries where there are universities.
*
* @returns {Array} of the countries instances
* @memberof Filter
*/
getCountriesWhereThereAreUniversities() {
const {mainCampuses, cities, countries} = this.getLatestReadDataFor(["mainCampuses", "cities", "countries"]);
let citiesMap = new Map(),
countriesMap = new Map();
cities.forEach(el => citiesMap.set(el.id, el));
countries.forEach(el => countriesMap.set(el.id, el));
static univHandler = undefined;
let res = new Map();
static values = {
countries: [],
semesters: [],
majorMinors: [],
};
mainCampuses.forEach(el => {
const cityId = el.city,
city = citiesMap.get(cityId),
countryId = city.country,
country = countriesMap.get(countryId);
componentWillUnmount() {
Filter.univHandler = undefined;
}
res.set(countryId, country);
});
return [...res.values()];
/**
* @return {FilterHandler}
*/
get univHandler() {
if (typeof Filter.univHandler === "undefined") {
Filter.univHandler = new FilterHandler(this.getLatestReadDataFor(["universities", "countries", "cities", "mainCampuses"]));
}
return Filter.univHandler;
}
updateSelectedUniversities(key, selection) {
Filter.values[key] = selection;
updateSelectedUniversities(selection) {
const mainCampuses = this.getLatestReadData("mainCampuses");
const {values} = Filter;
const selectedUniversities = this.univHandler.getSelection(values.countries, values.semesters, values.majorMinors);
let selectedUniversities = [];
mainCampuses.forEach(campus => {
const campusFull = this.joinCampus(campus);
if (selection.includes(campusFull.country.iso_alpha2_code)) {
selectedUniversities.push(campusFull.university.id);
}
});
this.props.saveSelection(selectedUniversities);
Filter.hasSelection = selectedUniversities.length !== 0;
Filter.hasSelection = Object.values(Filter.values).some(arr => arr.length !== 0);
Filter.nbSelection = selectedUniversities.length;
this.props.saveSelection(Filter.hasSelection ? selectedUniversities : null);
this.forceUpdate();
}
getEndMessage() {
if (!Filter.hasSelection) return "(Aucun filtre est actif)";
const base = "(Un filtre est actif — ";
if (Filter.nbSelection === 0) return base + "aucune université ne correspond)";
else if (Filter.nbSelection === 1) return base + "1 université correspond)";
else return base + `${Filter.nbSelection} universités correspondent)`;
}
customRender() {
const options = this.getCountriesWhereThereAreUniversities()
.map(c => ({value: c.iso_alpha2_code, label: c.name}));
const countriesOptions = this.univHandler.countriesOptions,
majorMinorOptions = this.univHandler.majorMinorOptions,
semestersOptions = this.univHandler.semesterOptions;
const {classes} = this.props;
return (
<ExpansionPanel expanded={Filter.isOpen} onChange={() => {
Filter.isOpen = !Filter.isOpen;
<ExpansionPanel expanded={Filter.isOpened} onChange={() => {
Filter.isOpened = !Filter.isOpened;
this.forceUpdate();
}}>
<ExpansionPanelSummary expandIcon={<ExpandMoreIcon/>}>
......@@ -93,17 +309,46 @@ class Filter extends CustomComponentForAPI {