From 00550d510873a067a4610d8627aab2035c93a55a Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Sun, 24 May 2020 21:40:54 +0200 Subject: [PATCH 01/34] style: missed previous reformat --- backend/backend_app/models/exchangeFeedback.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/backend_app/models/exchangeFeedback.py b/backend/backend_app/models/exchangeFeedback.py index 738ec387..7c42e433 100644 --- a/backend/backend_app/models/exchangeFeedback.py +++ b/backend/backend_app/models/exchangeFeedback.py @@ -85,12 +85,16 @@ class CharInFilter(filters.BaseInFilter, filters.CharFilter): class ExchangeFeedbackFilter(filters.FilterSet): - exchange__student_major_in = CharInFilter(field_name='exchange__student_major', lookup_expr='in') - exchange__student_minor_in = CharInFilter(field_name='exchange__student_minor', lookup_expr='in') + exchange__student_major_in = CharInFilter( + field_name="exchange__student_major", lookup_expr="in" + ) + exchange__student_minor_in = CharInFilter( + field_name="exchange__student_minor", lookup_expr="in" + ) class Meta: model = ExchangeFeedback - fields = ['exchange__student_major_in', 'exchange__student_minor_in'] + fields = ["exchange__student_major_in", "exchange__student_minor_in"] class ExchangeFeedbackFilterBackend(DjangoFilterBackend): @@ -116,10 +120,7 @@ class ExchangeFeedbackViewSet(EssentialModuleViewSet): ) serializer_class = ExchangeFeedbackSerializer end_point_route = "exchangeFeedbacks" - filterset_fields = ( - "university", - "untouched", - ) - filter_backends = (filters.DjangoFilterBackend, ExchangeFeedbackFilterBackend,) + filterset_fields = ("university", "untouched") + filter_backends = (filters.DjangoFilterBackend, ExchangeFeedbackFilterBackend) required_filterset_fields = ("university",) pagination_class = CustomPagination -- GitLab From 4325506e7378c8163591ad9d275b56c3f5f1f895 Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Sun, 24 May 2020 21:42:02 +0200 Subject: [PATCH 02/34] fix(univ form): marked readonly fields in UniversitySerializer --- CHANGELOG.md | 2 +- backend/backend_app/models/university.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aab4469..2303fe89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ###### _TBD_ -- [feat] blablabla +- [fix] front: univ form was broken (updated backend serializer) ## v2.6.0 diff --git a/backend/backend_app/models/university.py b/backend/backend_app/models/university.py index dd346a96..091908b9 100644 --- a/backend/backend_app/models/university.py +++ b/backend/backend_app/models/university.py @@ -64,6 +64,13 @@ class UniversitySerializer(EssentialModuleSerializer): "main_campus_lon", "denormalized_infos", ) + read_only_fields = ( + "city", + "country", + "main_campus_lat", + "main_campus_lon", + "denormalized_infos", + ) class UniversityViewSet(EssentialModuleViewSet): -- GitLab From 727b2df00d44845820ba8cb7d32c8a5f00c6cf1f Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Fri, 29 May 2020 20:43:25 +0200 Subject: [PATCH 03/34] feat(back/front): setup stats entry --- .prettierignore | 1 + backend/base_app/templates/stats.html | 11 ++++++ backend/base_app/urls.py | 1 + backend/base_app/views.py | 9 +++++ frontend/src/components/pages/PageStats.jsx | 21 ++++++++++++ frontend/src/entry/statsApp.jsx | 37 +++++++++++++++++++++ frontend/webpack.config.base.js | 1 + 7 files changed, 81 insertions(+) create mode 100644 .prettierignore create mode 100644 backend/base_app/templates/stats.html create mode 100644 frontend/src/components/pages/PageStats.jsx create mode 100644 frontend/src/entry/statsApp.jsx diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..2d19fc76 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.html diff --git a/backend/base_app/templates/stats.html b/backend/base_app/templates/stats.html new file mode 100644 index 00000000..75894277 --- /dev/null +++ b/backend/base_app/templates/stats.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} +{% load render_bundle from webpack_loader %} + +{% block content %} + + + {% render_bundle 'stats' %} + +{% endblock %} diff --git a/backend/base_app/urls.py b/backend/base_app/urls.py index 16e61b92..cba8151a 100644 --- a/backend/base_app/urls.py +++ b/backend/base_app/urls.py @@ -26,6 +26,7 @@ urlpatterns += [ name="cas_ng_proxy_callback", ), url(r"^app/.*", views.index), + url(r"^stats/.*", views.stats), url(r"^cgu-rgpd/.*", views.cgu_rgpd), url(r"^rgpd-raw/.*", views.rgpd_raw), url(r"^banned_note/", views.banned), diff --git a/backend/base_app/views.py b/backend/base_app/views.py index 54d2e9cf..cc7493f9 100644 --- a/backend/base_app/views.py +++ b/backend/base_app/views.py @@ -1,4 +1,5 @@ import logging +import json from django.http import HttpResponse, HttpResponseRedirect, HttpResponseNotFound from django.shortcuts import render @@ -38,6 +39,14 @@ def index(request): ) +def stats(request): + """ + Render the view that displays stats + """ + stats_data = [] + return render(request, "stats.html", dict(stats_data=json.dumps(stats_data))) + + def rgpd_raw(request): """ Render the view that displays only the RGPD conditions diff --git a/frontend/src/components/pages/PageStats.jsx b/frontend/src/components/pages/PageStats.jsx new file mode 100644 index 00000000..a4d5e936 --- /dev/null +++ b/frontend/src/components/pages/PageStats.jsx @@ -0,0 +1,21 @@ +import React from "react"; +import { compose } from "recompose"; +import Typography from "@material-ui/core/Typography"; +import { withPaddedPaper } from "./shared"; + +/** + * Component corresponding to the stats page of the site + */ +function PageStats() { + return ( + <> + Vive les stats +

+ More to come... + + ); +} + +PageStats.propTypes = {}; + +export default compose(withPaddedPaper())(PageStats); diff --git a/frontend/src/entry/statsApp.jsx b/frontend/src/entry/statsApp.jsx new file mode 100644 index 00000000..2eb40721 --- /dev/null +++ b/frontend/src/entry/statsApp.jsx @@ -0,0 +1,37 @@ +import React from "react"; +import ReactDOM from "react-dom"; +import CssBaseline from "@material-ui/core/CssBaseline"; +import OfflineThemeProvider from "../components/common/theme/OfflineThemeProvider"; +import Logo from "../components/app/Logo"; +import BaseTemplate from "../components/app/BaseTemplate"; +import PageStats from "../components/pages/PageStats"; + +function MainReactEntry() { + const toolbarContent = ( +
+ +
+ ); + + return ( + + <> + +
+ + + +
+ + + ); +} + +const wrapper = document.getElementById("app"); +ReactDOM.render(, wrapper); + +// eslint-disable-next-line no-undef +if (module.hot) { + // eslint-disable-next-line no-undef + module.hot.accept(); +} diff --git a/frontend/webpack.config.base.js b/frontend/webpack.config.base.js index 710ba7c5..ef6f68ed 100644 --- a/frontend/webpack.config.base.js +++ b/frontend/webpack.config.base.js @@ -5,6 +5,7 @@ const CopyPlugin = require("copy-webpack-plugin"); const config = { entry: { main: ["./src/entry/mainApp.jsx"], + stats: ["./src/entry/statsApp.jsx"], rgpdCgu: ["./src/entry/rgpdCguForm.jsx"], rgpdRaw: ["./src/entry/rgpdRaw.jsx"], }, -- GitLab From 82661796f421ebec7eb23d78e550df055a6df5be Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Fri, 29 May 2020 21:02:40 +0200 Subject: [PATCH 04/34] feat(front): added alasql and chart.js deps => new frontend image --- .gitlab-ci.yml | 6 +- docker-compose.yml | 2 +- frontend/package.json | 3 + frontend/src/components/pages/PageStats.jsx | 36 +++- frontend/yarn.lock | 189 +++++++++++++++++--- server/docker-compose.prod.yml | 2 +- 6 files changed, 209 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58054403..68d8c203 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,7 @@ check_back: check_front: <<: *only-default stage: check - image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.0.0 + image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.1.0 before_script: - cd frontend && mkdir -p node_modules && mv -f /usr/src/deps/node_modules/* /usr/src/deps/node_modules/.bin ./node_modules/ script: @@ -78,7 +78,7 @@ test_back: test_frontend: <<: *only-default stage: test - image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.0.0 + image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.1.0 before_script: - cd frontend && mkdir -p node_modules && mv -f /usr/src/deps/node_modules/* /usr/src/deps/node_modules/.bin ./node_modules/ script: @@ -98,7 +98,7 @@ flake8: eslint: <<: *only-default stage: lint - image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.0.0 + image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.1.0 before_script: - cd frontend && mkdir -p node_modules && mv -f /usr/src/deps/node_modules/* /usr/src/deps/node_modules/.bin ./node_modules/ script: diff --git a/docker-compose.yml b/docker-compose.yml index 7979b541..d6782f55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,7 +68,7 @@ services: # Service to handle frontend live developpments and building frontend: # Get the image from the registry - image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.0.0 + image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.1.0 # To use a locally built one, comment above, uncomment bellow. # build: ./frontend # On startup, we retrieve the dependencies from the image and start the developpement server diff --git a/frontend/package.json b/frontend/package.json index 1223a3eb..7d8494e0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -34,7 +34,9 @@ "@material-ui/lab": "^4.0.0-alpha.53", "@material-ui/pickers": "^3.2.4", "@material-ui/styles": "^4.9.14", + "alasql": "0.6.1", "axios": "^0.19.0", + "chart.js": "2.9.3", "core-js": "^3.2.1", "date-fns": "^2.0.1", "downshift": "^5.3.0", @@ -46,6 +48,7 @@ "prop-types": "^15.7.2", "react": "^16.9.0", "react-awesome-slider": "^4.1.0", + "react-chartjs-2": "2.9.0", "react-dom": "^16.9.0", "react-mapbox-gl": "^4.6.0", "react-markdown": "^4.1.0", diff --git a/frontend/src/components/pages/PageStats.jsx b/frontend/src/components/pages/PageStats.jsx index a4d5e936..8f4b2a83 100644 --- a/frontend/src/components/pages/PageStats.jsx +++ b/frontend/src/components/pages/PageStats.jsx @@ -1,8 +1,21 @@ import React from "react"; import { compose } from "recompose"; import Typography from "@material-ui/core/Typography"; +import { Line } from "react-chartjs-2"; +import alasql from "alasql"; import { withPaddedPaper } from "./shared"; +const data = [ + { a: 1, b: 1, c: 1 }, + { a: 1, b: 2, c: 1 }, + { a: 1, b: 3, c: 1 }, + { a: 2, b: 1, c: 1 }, +]; +const res = alasql("SELECT a, COUNT(*) AS c FROM ? GROUP BY a", [data]); + +// eslint-disable-next-line no-console +console.log(res); + /** * Component corresponding to the stats page of the site */ @@ -10,8 +23,29 @@ function PageStats() { return ( <> Vive les stats -

+
More to come... + ); } diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 095867e0..948a80e2 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1650,6 +1650,14 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.2.0.tgz#17ea7e40d7c8640ff54a694c889c26f31704effe" integrity sha512-apwXVmYVpQ34m/i71vrApRrRKCWQnZZF1+npOD0WV5xZFfwWOmKGQ2RWlfdy9vWITsenisM8M0Qeq8agcFHNiQ== +adler-32@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25" + integrity sha1-aj5r8KY5ALoVZSgIyxXGgT0aXyU= + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + ajv-errors@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d" @@ -1670,6 +1678,18 @@ ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.5.5: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +alasql@0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/alasql/-/alasql-0.6.1.tgz#a712a3b4baf14e02b3ceae6adfba3a6c6cb1d0ad" + integrity sha512-MUsAKQNbCJqBEWTw4buRyN5tqiugZKhm2ombq7XO9cDGPDyjcCJURtc/dTnHOKEZ+ZkTzKhmUSGlpIq7VprKxQ== + dependencies: + dom-storage "^2.1.0" + es6-promise "^4.2.6" + lodash "^4.17.11" + request "2.88.2" + xlsx "0.16.0" + yargs "15.3.1" + amdefine@>=0.0.4: version "1.0.1" resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5" @@ -2414,6 +2434,16 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= +cfb@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.1.4.tgz#81fd35ede4c919d8f0962a94582e1dfaf7051e2a" + integrity sha512-rwFkl3aFO3f+ljR27YINwC0x8vPjyiEVbYbrTCKzspEf7Q++3THdfHVgJYNUbxNcupJECrLX+L40Mjm9hm/Bgw== + dependencies: + adler-32 "~1.2.0" + commander "^2.16.0" + crc-32 "~1.2.0" + printj "~1.1.2" + chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -2480,6 +2510,29 @@ chardet@^0.7.0: resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== +chart.js@2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7" + integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw== + dependencies: + chartjs-color "^2.1.0" + moment "^2.10.2" + +chartjs-color-string@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71" + integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== + dependencies: + color-name "^1.0.0" + +chartjs-color@^2.1.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0" + integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== + dependencies: + chartjs-color-string "^0.6.0" + color-convert "^1.9.3" + check-types@^8.0.3: version "8.0.3" resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552" @@ -2593,6 +2646,14 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= +codepage@~1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99" + integrity sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k= + dependencies: + commander "~2.14.1" + exit-on-epipe "~1.0.1" + collapse-white-space@^1.0.2: version "1.0.6" resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-1.0.6.tgz#e63629c0016665792060dbbeb79c42239d2c5287" @@ -2611,7 +2672,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0: +color-convert@^1.9.0, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -2630,7 +2691,7 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= -color-name@~1.1.4: +color-name@^1.0.0, color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== @@ -2642,11 +2703,21 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -commander@^2.11.0, commander@^2.18.0, commander@^2.20.0: +commander@^2.11.0, commander@^2.16.0, commander@^2.18.0, commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== +commander@~2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa" + integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw== + +commander@~2.17.1: + version "2.17.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf" + integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg== + commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" @@ -2845,6 +2916,14 @@ cosmiconfig@^6.0.0: path-type "^4.0.0" yaml "^1.7.2" +crc-32@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208" + integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA== + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + create-ecdh@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff" @@ -3274,6 +3353,11 @@ dom-serializer@^0.2.1: domelementtype "^2.0.1" entities "^2.0.0" +dom-storage@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/dom-storage/-/dom-storage-2.1.0.tgz#00fb868bc9201357ea243c7bcfd3304c1e34ea39" + integrity sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q== + dom-walk@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84" @@ -3480,6 +3564,11 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" +es6-promise@^4.2.6: + version "4.2.8" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a" + integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w== + escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" @@ -3801,6 +3890,11 @@ execa@^4.0.0: signal-exit "^3.0.2" strip-final-newline "^2.0.0" +exit-on-epipe@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" + integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw== + exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -4159,6 +4253,11 @@ forwarded@~0.1.2: resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ= +frac@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b" + integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA== + fragment-cache@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" @@ -6042,7 +6141,7 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= -lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@~4.17.12: +lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@~4.17.12: version "4.17.15" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== @@ -6386,6 +6485,11 @@ mixin-deep@^1.2.0: dependencies: minimist "^1.2.5" +moment@^2.10.2: + version "2.26.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.26.0.tgz#5e1f82c6bafca6e83e808b30c8705eed0dcbd39a" + integrity sha512-oIixUO+OamkUkwjhAVE18rAMfRJNsNe/Stid/gwHSOfHrOtw9EhAY2AHvdKZ/k/MggcYELFCJz/Sn2pL8b8JMw== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -7381,6 +7485,11 @@ pretty-quick@^2.0.1: mri "^1.1.4" multimatch "^4.0.0" +printj@~1.1.0, printj@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" + integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== + private@^0.1.8: version "0.1.8" resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff" @@ -7421,7 +7530,7 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.4" -prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2: +prop-types@^15.5.8, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2: version "15.7.2" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.7.2.tgz#52c41e75b8c87e72b9d9360e0206b99dcbffa6c5" integrity sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ== @@ -7598,6 +7707,14 @@ react-awesome-slider@^4.1.0: dependencies: web-animation-club "^0.6.0" +react-chartjs-2@2.9.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/react-chartjs-2/-/react-chartjs-2-2.9.0.tgz#d054dbdd763fbe9a76296a4ae0752ea549b76d9e" + integrity sha512-IYwqUUnQRAJ9SNA978vxulHJTcUFTJk2LDVfbAyk0TnJFZZG7+6U/2flsE4MCw6WCbBjTTypy8T82Ch7XrPtRw== + dependencies: + lodash "^4.17.4" + prop-types "^15.5.8" + react-dom@^16.9.0: version "16.13.1" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.13.1.tgz#c1bd37331a0486c078ee54c4740720993b2e0e7f" @@ -7958,7 +8075,7 @@ request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@^2.87.0, request@^2.88.0, request@^2.88.2: +request@2.88.2, request@^2.87.0, request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -8626,6 +8743,13 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +ssf@~0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/ssf/-/ssf-0.10.3.tgz#8eae1fc29c90a552e7921208f81892d6f77acb2b" + integrity sha512-pRuUdW0WwyB2doSqqjWyzwCD6PkfxpHAHdZp39K3dp/Hq7f+xfMwNAWIi16DyrRg4gg9c/RvLYkJTSawTPTm1w== + dependencies: + frac "~1.1.2" + sshpk@^1.7.0: version "1.16.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" @@ -9828,6 +9952,11 @@ wide-align@^1.1.0: dependencies: string-width "^1.0.2 || 2" +wmf@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wmf/-/wmf-1.0.2.tgz#7d19d621071a08c2bdc6b7e688a9c435298cc2da" + integrity sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw== + word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" @@ -9897,6 +10026,20 @@ x-is-string@^0.1.0: resolved "https://registry.yarnpkg.com/x-is-string/-/x-is-string-0.1.0.tgz#474b50865af3a49a9c4657f05acd145458f77d82" integrity sha1-R0tQhlrzpJqcRlfwWs0UVFj3fYI= +xlsx@0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.16.0.tgz#2a86ba433ca198f7e87343da86a831d48d713943" + integrity sha512-W/LQZjh6o7WDGmCIUXp2FUPSej2XRdaiTgZN31Oh68JlL7jpm796p3eI5zOpphYNT12qkgnXYaCysAsoiyZvOQ== + dependencies: + adler-32 "~1.2.0" + cfb "^1.1.4" + codepage "~1.14.0" + commander "~2.17.1" + crc-32 "~1.2.0" + exit-on-epipe "~1.0.1" + ssf "~0.10.3" + wmf "~1.0.1" + xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" @@ -9974,23 +10117,7 @@ yargs@13.2.4: y18n "^4.0.0" yargs-parser "^13.1.0" -yargs@^13.3.2: - version "13.3.2" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" - integrity sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw== - dependencies: - cliui "^5.0.0" - find-up "^3.0.0" - get-caller-file "^2.0.1" - require-directory "^2.1.1" - require-main-filename "^2.0.0" - set-blocking "^2.0.0" - string-width "^3.0.0" - which-module "^2.0.0" - y18n "^4.0.0" - yargs-parser "^13.1.2" - -yargs@^15.3.1: +yargs@15.3.1, yargs@^15.3.1: version "15.3.1" resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.3.1.tgz#9505b472763963e54afe60148ad27a330818e98b" integrity sha512-92O1HWEjw27sBfgmXiixJWT5hRBp2eobqXicLtPBIDBhYB+1HpwZlXmbW2luivBJHBzki+7VyCLRtAkScbTBQA== @@ -10006,3 +10133,19 @@ yargs@^15.3.1: which-module "^2.0.0" y18n "^4.0.0" yargs-parser "^18.1.1" + +yargs@^13.3.2: + version "13.3.2" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" + integrity sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw== + dependencies: + cliui "^5.0.0" + find-up "^3.0.0" + get-caller-file "^2.0.1" + require-directory "^2.1.1" + require-main-filename "^2.0.0" + set-blocking "^2.0.0" + string-width "^3.0.0" + which-module "^2.0.0" + y18n "^4.0.0" + yargs-parser "^13.1.2" diff --git a/server/docker-compose.prod.yml b/server/docker-compose.prod.yml index a5cef146..2281c6f1 100644 --- a/server/docker-compose.prod.yml +++ b/server/docker-compose.prod.yml @@ -39,7 +39,7 @@ services: volumes: ["postgres_data_prod:/var/lib/postgresql/data/"] frontend: # Will be killed as soon as the front is generated - image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.0.0 + image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v2.1.0 command: /bin/sh -c "cd frontend && mv -f /usr/src/deps/node_modules/* /usr/src/deps/node_modules/.bin ./node_modules/ && yarn build" networks: [] volumes: -- GitLab From c47e58bdbf2779f6046296432575283b4fd793a2 Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Fri, 29 May 2020 21:04:22 +0200 Subject: [PATCH 05/34] doc: updated changelog Relates to #192 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2303fe89..b13f2d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ###### _TBD_ - [fix] front: univ form was broken (updated backend serializer) +- [feat] front / back: added stats view and frontend deps needed for the view ## v2.6.0 -- GitLab From 37062c649fad7aaf07faeff5d8cee202b41c8544 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 17:56:45 +0200 Subject: [PATCH 06/34] feat(backend): create custom admin site --- backend/base_app/admin.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/backend/base_app/admin.py b/backend/base_app/admin.py index 321fa39a..e77eb93d 100644 --- a/backend/base_app/admin.py +++ b/backend/base_app/admin.py @@ -2,15 +2,31 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin from django.contrib.auth.forms import UserChangeForm from rest_framework.authtoken.admin import TokenAdmin +from django.urls import path -from .models import User +from base_app.models import User + +# create a custom admin site +from base_app.views import trigger_cron + + +class CustomAdminSite(admin.AdminSite): + """ + Custom admin site used to add a trigger_cron view + on the admin site provided by django + """ + def get_urls(self): + urls = super().get_urls() + urls += [path("trigger_cron/", self.admin_view(trigger_cron))] + return urls + + +admin_site = CustomAdminSite(name="custom_admin_site") # Handling of the registration of the custom User model to make sure # we can see all the fields. # taken from: https://stackoverflow.com/a/15013810 - - class CustomUserChangeForm(UserChangeForm): class Meta(UserChangeForm.Meta): model = User @@ -27,7 +43,7 @@ class CustomUserAdmin(UserAdmin): ) -admin.site.register(User, CustomUserAdmin) +admin_site.register(User, CustomUserAdmin) # Pour la génération de token dans l'administration du site. TokenAdmin.raw_id_fields = ("user",) -- GitLab From f0be1be1d8aebeb7abacda028dd5082dc1ff71de Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 17:57:55 +0200 Subject: [PATCH 07/34] feat(backend): copy cron tasks without decorators to be executed manually --- backend/_cron_tasks.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 backend/_cron_tasks.py diff --git a/backend/_cron_tasks.py b/backend/_cron_tasks.py new file mode 100644 index 00000000..3603a1a2 --- /dev/null +++ b/backend/_cron_tasks.py @@ -0,0 +1,31 @@ +from backend_app.models.exchange import update_denormalized_univ_major_minor + +from backend_app.models.university import update_denormalized_univ_field # noqa: E402 +from external_data.management.commands.utils import FixerData, UtcData # noqa: E402 +from base_app.management.commands.clean_user_accounts import ( + ClearUserAccounts, + ClearSessions, +) +from stats_app.compute_stats import update_all_stats + + +def update_currencies(): + FixerData().update() + + +def update_utc_ent(): + UtcData().update() + + +def update_extra_denormalization(): + update_denormalized_univ_major_minor() + update_denormalized_univ_field() + + +def clear_and_clean_sessions(): + ClearUserAccounts.run() + ClearSessions.run() + + +def update_daily_stats(): + update_all_stats() -- GitLab From ad1f2655b03c449122de1d27a0aa7445e410aaa8 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:00:07 +0200 Subject: [PATCH 08/34] fix(backend): add missing argument num to the new cron task --- backend/cron_tasks.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/backend/cron_tasks.py b/backend/cron_tasks.py index 03bb0c49..12420f81 100644 --- a/backend/cron_tasks.py +++ b/backend/cron_tasks.py @@ -11,17 +11,15 @@ django.setup() import logging # noqa: E402 from uwsgidecorators import harakiri, cron, timer # noqa: E402 -from backend_app.models.exchange import ( - update_denormalized_univ_major_minor, -) # noqa: E402 -from backend_app.models.university import update_denormalized_univ_field # noqa: E402 -from external_data.management.commands.utils import FixerData, UtcData # noqa: E402 -from base_app.management.commands.clean_user_accounts import ( - ClearUserAccounts, - ClearSessions, +from _cron_tasks import ( + clear_and_clean_sessions, + update_daily_stats, + update_currencies, + update_extra_denormalization, + update_utc_ent, ) # noqa: E402 -from stats_app.compute_stats import update_all_stats # noqa: E402 + logger = logging.getLogger("django") @@ -29,31 +27,29 @@ logger = logging.getLogger("django") # @timer(60, target="spooler") # use this for tests @timer(3 * 60 * 60, target="spooler") # run it three hours @harakiri(40) -def update_currencies(num): - FixerData().update() +def _update_currencies(num): + update_currencies() @cron(5, 0, -1, -1, -1, target="spooler") # everyday at 5 past midnight @harakiri(60 * 10) # shouldn't take more than 10 minutes to run -def update_utc_ent(num): - UtcData().update() +def _update_utc_ent(num): + update_utc_ent() @timer(60 * 60, target="spooler") # run it every hour @harakiri(60) -def update_extra_denormalization(num): - update_denormalized_univ_major_minor() - update_denormalized_univ_field() +def _update_extra_denormalization(num): + update_extra_denormalization() @cron(20, 0, -1, -1, -1, target="spooler") # everyday at 20 past midnight @harakiri(60 * 5) # shouldn't take more than 5 minutes to run -def clear_and_clean_sessions(num): - ClearUserAccounts.run() - ClearSessions.run() +def _clear_and_clean_sessions(num): + clear_and_clean_sessions() @cron(30, 0, -1, -1, -1, target="spooler") # everyday at 30 past midnight @harakiri(60 * 5) # shouldn't take more than 5 minutes to run -def update_daily_stats(): - update_all_stats() +def _update_daily_stats(num): + update_daily_stats() -- GitLab From c61d56017d64aa28152e5aa0375cf06dd8f88449 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:02:26 +0200 Subject: [PATCH 09/34] feat(backend): create custom admin view and register it --- backend/base_app/urls.py | 4 ++-- backend/base_app/views.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/base_app/urls.py b/backend/base_app/urls.py index cba8151a..30d63e96 100644 --- a/backend/base_app/urls.py +++ b/backend/base_app/urls.py @@ -1,10 +1,10 @@ import django_cas_ng.views from django.conf import settings from django.conf.urls import include, url -from django.contrib import admin from django.views.generic.base import RedirectView from base_app.views import media_files_view +from base_app.admin import admin_site from . import views if settings.DEBUG: @@ -15,7 +15,7 @@ else: urlpatterns = [] urlpatterns += [ - url(r"^admin/", admin.site.urls), + url(r"^admin/", admin_site.urls), url(r"^user/login$", django_cas_ng.views.LoginView.as_view(), name="cas_ng_login"), url( r"^user/logout$", django_cas_ng.views.LogoutView.as_view(), name="cas_ng_logout" diff --git a/backend/base_app/views.py b/backend/base_app/views.py index cc7493f9..306aea25 100644 --- a/backend/base_app/views.py +++ b/backend/base_app/views.py @@ -9,6 +9,7 @@ from backend_app.utils import clean_route from backend_app.viewsets import ALL_VIEWSETS from base_app.forms import UserForm from base_app.models import User +from _cron_tasks import update_currencies, update_utc_ent, update_extra_denormalization, clear_and_clean_sessions logger = logging.getLogger("django") @@ -94,3 +95,23 @@ def media_files_view(request, path): del response["Content-Type"] response["X-Accel-Redirect"] = "/protected-assets/media/" + path return response + + +def trigger_cron(request): + """ + Render the view that displays cron tasks + """ + if request.method == "POST": + cron_task = request.POST.get("cron_name") + if cron_task == "update_currencies": + update_currencies() + elif cron_task == "update_utc_ent": + update_utc_ent() + elif cron_task == "update_extra_denormalization": + update_extra_denormalization() + elif cron_task == "clear_and_clean_sessions": + clear_and_clean_sessions() + else: + return HttpResponseNotFound() + + return render(request, "admin/trigger_cron.html") -- GitLab From 87b69d2a8c2285246c83a954d612b04c702d6389 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:03:10 +0200 Subject: [PATCH 10/34] feat(backend): create template for custom admin view --- .../templates/admin/trigger_cron.html | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 backend/base_app/templates/admin/trigger_cron.html diff --git a/backend/base_app/templates/admin/trigger_cron.html b/backend/base_app/templates/admin/trigger_cron.html new file mode 100644 index 00000000..b7c01965 --- /dev/null +++ b/backend/base_app/templates/admin/trigger_cron.html @@ -0,0 +1,25 @@ +{% extends "admin/index.html" %} {% block content %} +
+ {% csrf_token %} +
+ +
+ +
+ {% csrf_token %} +
+ +
+ +
+ {% csrf_token %} +
+ +
+ +
+ {% csrf_token %} +
+ +
+{% endblock %} -- GitLab From 2fb16ed0ad77228ae0536c231ea0305225c9e936 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:04:33 +0200 Subject: [PATCH 11/34] feat(backend): register views in custom admin site --- backend/backend_app/admin.py | 6 +++--- backend/backend_app/viewsets.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/backend_app/admin.py b/backend/backend_app/admin.py index 702a7f8a..22fdbacb 100644 --- a/backend/backend_app/admin.py +++ b/backend/backend_app/admin.py @@ -1,4 +1,4 @@ -from django.contrib import admin +from base_app.admin import admin_site from reversion_compare.admin import CompareVersionAdmin from backend_app.models.abstract.versionedEssentialModule import ( @@ -73,8 +73,8 @@ VERSIONED_MODELS = filter(lambda m: issubclass(m, VersionedEssentialModule), ALL for Model in CLASSIC_MODELS: # Register the model in the admin in a standard way - admin.site.register(Model) + admin_site.register(Model) for Model in VERSIONED_MODELS: # Register the model in the admin with versioning - admin.site.register(Model, CompareVersionAdmin) + admin_site.register(Model, CompareVersionAdmin) diff --git a/backend/backend_app/viewsets.py b/backend/backend_app/viewsets.py index 6cdc6c70..0ba75c49 100644 --- a/backend/backend_app/viewsets.py +++ b/backend/backend_app/viewsets.py @@ -212,7 +212,9 @@ class UnlinkedUtcPartners(ViewSet): def list(self, request): partners = Partner.objects.filter(university=None) - return Response(list(map(lambda partner: partner.univ_name, partners))) + return Response( + list(map(lambda partner: (partner.utc_id, partner.univ_name), partners)) + ) class UpdateStudentExchangesViewSet(ViewSet): -- GitLab From 33c21faecc272fa843bf0bb6797ccc2849059549 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:05:28 +0200 Subject: [PATCH 12/34] fix(backend): register stats to custom admin site --- backend/stats_app/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/stats_app/admin.py b/backend/stats_app/admin.py index f81e619d..ae9c8989 100644 --- a/backend/stats_app/admin.py +++ b/backend/stats_app/admin.py @@ -1,5 +1,5 @@ -from django.contrib import admin +from base_app.admin import admin_site from stats_app.models import DailyConnections -admin.site.register(DailyConnections) +admin_site.register(DailyConnections) -- GitLab From 8f29437a16baf02e6458e5779449e21340887423 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:07:02 +0200 Subject: [PATCH 13/34] docs(backend): add custom page to doc sidebar --- documentation/_sidebar.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/_sidebar.md b/documentation/_sidebar.md index ac5fa522..54103389 100644 --- a/documentation/_sidebar.md +++ b/documentation/_sidebar.md @@ -39,9 +39,10 @@ - [Architecture](ProdRelated/architecture.md) - [Dump database](ProdRelated/dump_database.md) - - Maintenance - - [Update major minor](ProdRelated/updateMajorMinor.md) + - Maintenance + - [Linking partners / universities](ProdRelated/maintenance/linking_universities.md) + - [Update major minor](ProdRelated/maintenance/update_major_minor.md) * Comments about technologies used -- GitLab From 1ea346e4a7d975488e11f3643d95e4bce64b3b4e Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:08:01 +0200 Subject: [PATCH 14/34] docs(backend): create new folder in doc to contain linking universities process --- .../maintenance/linking_universities.md | 18 ++++++++++++++++++ .../maintenance/update_major_minor.md | 17 +++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 documentation/ProdRelated/maintenance/linking_universities.md create mode 100644 documentation/ProdRelated/maintenance/update_major_minor.md diff --git a/documentation/ProdRelated/maintenance/linking_universities.md b/documentation/ProdRelated/maintenance/linking_universities.md new file mode 100644 index 00000000..7b4c442a --- /dev/null +++ b/documentation/ProdRelated/maintenance/linking_universities.md @@ -0,0 +1,18 @@ +# Linking Universities + +## Problem description + +Data from UTC DSI is automatically updated every day. +However, we need to link new university data from the DSI with universities on REX-DRI to guarantee data integrity (there might be duplicates on the DSI side) and correctness. Unlinked universities ids and names are displayed on this route: [https://rex.dri.utc.fr/app/about/unlinked-universities](https://rex.dri.utc.fr/app/about/unlinked-universities). + +## Linking (DSI) partners and (REX-DRI) universities + +There are two cases: + +1. The university (on REX-DRI) already exists. Go to [https://rex.dri.utc.fr/admin](https://rex.dri.utc.fr/admin), select the Partner object with the correct id. Then link it to the right university object. + +2. The university does not exist. Go to [https://rex.dri.utc.fr/admin](https://rex.dri.utc.fr/admin) create a new University object, with all its information (website, coordinates, etc.). Then select the Partner object with the corresponding id and link it with the university just created (just like in case 1.). + +## Updating denormalized information + +After you have done either 1. or 2., you **must** update the denormalized information on the universities. Administrators can do it from the trigger_cron admin page: [https://rex.dri.utc.fr/admin/trigger_cron](https://rex.dri.utc.fr/admin/trigger_cron). Click on update extra denormalization to update information about universities. diff --git a/documentation/ProdRelated/maintenance/update_major_minor.md b/documentation/ProdRelated/maintenance/update_major_minor.md new file mode 100644 index 00000000..1f0040e0 --- /dev/null +++ b/documentation/ProdRelated/maintenance/update_major_minor.md @@ -0,0 +1,17 @@ +# Update majors / minors + +In case of changes of majors / minors at the UTC, the frontend filter utils need to be updated. + +In `frontend/src/utils/majorMinorMappings.js`, each change has to be added as a new line in either `majorMapping` or `minorMapping` like this: + +```js +export const majorMapping = { + oldMajorName: "newMajorName" +}; + +export const minorMapping = { + majorName: { + oldMinorName: "newMinorName" + } +}; +``` -- GitLab From 394329bea073ac7760a7238957c7bf25befa2245 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:08:58 +0200 Subject: [PATCH 15/34] fix(frontend): change frontend text --- documentation/ProdRelated/updateMajorMinor.md | 17 ----------------- .../pages/PageAboutUnlinkedPartners.jsx | 4 ++-- 2 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 documentation/ProdRelated/updateMajorMinor.md diff --git a/documentation/ProdRelated/updateMajorMinor.md b/documentation/ProdRelated/updateMajorMinor.md deleted file mode 100644 index c5a4f1db..00000000 --- a/documentation/ProdRelated/updateMajorMinor.md +++ /dev/null @@ -1,17 +0,0 @@ -# Update majors / minors - -In case of changes of majors / minors at the UTC, the frontend filter utils need to be updated. - -In `frontend/src/utils/majorMinorMappings.js`, each change has to be added as a new line in either `majorMapping` or `minorMapping` like this: - -```js -export const majorMapping = { - oldMajorName: "newMajorName", -}; - -export const minorMapping = { - majorName: { - oldMinorName: "newMinorName", - }, -}; -``` diff --git a/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx b/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx index 08c8919e..2869a95e 100644 --- a/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx +++ b/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx @@ -11,8 +11,8 @@ automatiquement les données de l'UTC. Parmis ces données sont les partenaires Lorsque de nouveaux partenaires sont ajoutés par l'UTC, il est nécessaire de l'associer à différentes informations (localisation, site internet, etc.) du côté de la plateforme -**REX-DRI**. Cette opération doit se faire manuellement (n'hésitez pas à relancer le SIMDE -si cela tarde). +**REX-DRI**. Cette opération doit se faire manuellement. Le processus d'association est actuellement +en cours de simplification et les universités ci-dessous seront reliées prochainement. `; /** -- GitLab From 5e6c4d21d53b0364ab5001aa1ca3b15789478e33 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:21:15 +0200 Subject: [PATCH 16/34] fix(backend): add button for newly created cron task for stats --- backend/base_app/templates/admin/trigger_cron.html | 6 ++++++ backend/base_app/views.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/base_app/templates/admin/trigger_cron.html b/backend/base_app/templates/admin/trigger_cron.html index b7c01965..847156f2 100644 --- a/backend/base_app/templates/admin/trigger_cron.html +++ b/backend/base_app/templates/admin/trigger_cron.html @@ -22,4 +22,10 @@
+ +
+ {% csrf_token %} +
+ +
{% endblock %} diff --git a/backend/base_app/views.py b/backend/base_app/views.py index 306aea25..3293e7df 100644 --- a/backend/base_app/views.py +++ b/backend/base_app/views.py @@ -9,7 +9,13 @@ from backend_app.utils import clean_route from backend_app.viewsets import ALL_VIEWSETS from base_app.forms import UserForm from base_app.models import User -from _cron_tasks import update_currencies, update_utc_ent, update_extra_denormalization, clear_and_clean_sessions +from _cron_tasks import ( + update_currencies, + update_utc_ent, + update_extra_denormalization, + clear_and_clean_sessions, + update_daily_stats, +) logger = logging.getLogger("django") @@ -111,6 +117,8 @@ def trigger_cron(request): update_extra_denormalization() elif cron_task == "clear_and_clean_sessions": clear_and_clean_sessions() + elif cron_task == "update_daily_stats": + update_daily_stats() else: return HttpResponseNotFound() -- GitLab From 4de5d5732b834270a8425899d7ff25db74c58819 Mon Sep 17 00:00:00 2001 From: Gautier D Date: Sat, 30 May 2020 18:34:47 +0200 Subject: [PATCH 17/34] fix(docs): remove useless sentence --- .../src/components/pages/PageAboutUnlinkedPartners.jsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx b/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx index 2869a95e..71576ac2 100644 --- a/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx +++ b/frontend/src/components/pages/PageAboutUnlinkedPartners.jsx @@ -6,13 +6,12 @@ import { withPaddedPaper } from "./shared"; import UnlinkedPartners from "../app/UnlinkedPartners"; const source = ` -Pour tendre vers l'expérience la plus à jour **REX-DRI** récupère tous les jours +Pour tendre vers l'expérience la plus à jour **REX-DRI** récupère tous les jours automatiquement les données de l'UTC. Parmis ces données sont les partenaires de l'UTC. -Lorsque de nouveaux partenaires sont ajoutés par l'UTC, il est nécessaire de l'associer -à différentes informations (localisation, site internet, etc.) du côté de la plateforme -**REX-DRI**. Cette opération doit se faire manuellement. Le processus d'association est actuellement -en cours de simplification et les universités ci-dessous seront reliées prochainement. +Lorsque de nouveaux partenaires sont ajoutés par l'UTC, il est nécessaire de l'associer +à différentes informations (localisation, site internet, etc.) du côté de la plateforme +**REX-DRI**. Cette opération doit se faire manuellement. `; /** -- GitLab From 1b006a98160adb5b96e286e2889a1379b35ceb3f Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Sun, 31 May 2020 14:24:02 +0200 Subject: [PATCH 18/34] chore: cleaned show unlinked partners --- backend/backend_app/viewsets.py | 2 +- frontend/src/components/app/UnlinkedPartners.jsx | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/backend_app/viewsets.py b/backend/backend_app/viewsets.py index 0ba75c49..ae57d2ca 100644 --- a/backend/backend_app/viewsets.py +++ b/backend/backend_app/viewsets.py @@ -213,7 +213,7 @@ class UnlinkedUtcPartners(ViewSet): partners = Partner.objects.filter(university=None) return Response( - list(map(lambda partner: (partner.utc_id, partner.univ_name), partners)) + [dict(partner_id=p.utc_id, partner_univ_name=p.univ_name) for p in partners] ) diff --git a/frontend/src/components/app/UnlinkedPartners.jsx b/frontend/src/components/app/UnlinkedPartners.jsx index 23cf92cd..2f0d6343 100644 --- a/frontend/src/components/app/UnlinkedPartners.jsx +++ b/frontend/src/components/app/UnlinkedPartners.jsx @@ -21,14 +21,18 @@ function UnlinkedPartners({ unlinkedPartners, variant }) { {nUnlinked > 0 ? ( <> - {nUnlinked} + Actuellement, {nUnlinked}   partenaires sont dans ce cas. En voici la liste : - {unlinkedPartners.map((nameOnEnt, idx) => ( + {unlinkedPartners.map((p, idx) => ( // eslint-disable-next-line react/no-array-index-key - + ))} -- GitLab From 098a9e51e5ca13abb02868ead5f7d36934cd75ec Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Sun, 31 May 2020 14:26:43 +0200 Subject: [PATCH 19/34] docs: updated changelog Closes #187 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b13f2d4c..29771bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [fix] front: univ form was broken (updated backend serializer) - [feat] front / back: added stats view and frontend deps needed for the view +- [feat] Create a custom admin site containing a view to run cron tasks manually. ## v2.6.0 -- GitLab From 23db719f449996029e2b9bc7636c71061d3d7f80 Mon Sep 17 00:00:00 2001 From: Maxime Date: Sun, 7 Jun 2020 11:37:37 +0200 Subject: [PATCH 20/34] feature(backend): Add model and migration --- .../migrations/0002_auto_20200601_1418.py | 53 +++++++++++++++++++ backend/stats_app/models.py | 15 ++++++ 2 files changed, 68 insertions(+) create mode 100644 backend/stats_app/migrations/0002_auto_20200601_1418.py diff --git a/backend/stats_app/migrations/0002_auto_20200601_1418.py b/backend/stats_app/migrations/0002_auto_20200601_1418.py new file mode 100644 index 00000000..654ff3e5 --- /dev/null +++ b/backend/stats_app/migrations/0002_auto_20200601_1418.py @@ -0,0 +1,53 @@ +# Generated by Django 2.1.7 on 2020-06-01 12:18 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("backend_app", "0005_lastvisiteduniversity"), + ("stats_app", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="DailyExchangeContributionsInfo", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date", models.DateTimeField()), + ("major", models.CharField(blank=True, max_length=20)), + ("minor", models.CharField(blank=True, max_length=47)), + ("exchange_semester", models.CharField(max_length=5)), + ( + "nb_contributions", + models.IntegerField( + validators=[django.core.validators.MinValueValidator(0)] + ), + ), + ( + "university", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="backend_app.University", + ), + ), + ], + ), + migrations.AlterUniqueTogether( + name="dailyexchangecontributionsinfo", + unique_together={ + ("date", "university", "major", "minor", "exchange_semester") + }, + ), + ] diff --git a/backend/stats_app/models.py b/backend/stats_app/models.py index 8f59831d..f56ca5d8 100644 --- a/backend/stats_app/models.py +++ b/backend/stats_app/models.py @@ -1,7 +1,22 @@ from django.db import models from django.core.validators import MinValueValidator +from backend_app.models.university import University class DailyConnections(models.Model): date = models.DateTimeField(unique=True, null=False) nb_connections = models.IntegerField(validators=[MinValueValidator(0)], null=False) + + +class DailyExchangeContributionsInfo(models.Model): + date = models.DateTimeField(null=False) + university = models.ForeignKey(University, null=False, on_delete=models.CASCADE) + major = models.CharField(max_length=20, null=False, blank=True) + minor = models.CharField(max_length=47, null=False, blank=True) + exchange_semester = models.CharField(max_length=5, null=False) + nb_contributions = models.IntegerField( + validators=[MinValueValidator(0)], null=False + ) + + class Meta: + unique_together = ("date", "university", "major", "minor", "exchange_semester") -- GitLab From 1fc1f920bdbeb28f393bb9e19fce28e077cea4fb Mon Sep 17 00:00:00 2001 From: Maxime Date: Sun, 7 Jun 2020 11:43:41 +0200 Subject: [PATCH 21/34] feature(backend): Add compute functions and utils to get stats about contributions --- backend/stats_app/compute_stats.py | 30 +++++++++++++- backend/stats_app/utils.py | 63 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/backend/stats_app/compute_stats.py b/backend/stats_app/compute_stats.py index 3e77f425..d285d432 100644 --- a/backend/stats_app/compute_stats.py +++ b/backend/stats_app/compute_stats.py @@ -1,7 +1,14 @@ from datetime import timedelta +from collections import Counter +from backend_app.models.university import University -from stats_app.models import DailyConnections -from stats_app.utils import get_daily_connections, get_today_as_datetime + +from stats_app.models import DailyConnections, DailyExchangeContributionsInfo +from stats_app.utils import ( + get_daily_connections, + get_today_as_datetime, + get_contributions_profiles, +) def update_daily_connections(): @@ -11,5 +18,24 @@ def update_daily_connections(): ) +def update_daily_exchange_contributions_info(): + yesterday = get_today_as_datetime() - timedelta(days=1) + + contributions_profiles = get_contributions_profiles() + + nb_contributions_by_profile = Counter(contributions_profiles) + for contribution_profile, nb_contributions in nb_contributions_by_profile.items(): + university = University.objects.get(pk=contribution_profile.university_pk) + DailyExchangeContributionsInfo.objects.update_or_create( + date=yesterday, + major=contribution_profile.major, + minor=contribution_profile.minor, + exchange_semester=contribution_profile.exchange_semester, + university=university, + defaults=dict(nb_contributions=nb_contributions), + ) + + def update_all_stats(): update_daily_connections() + update_daily_exchange_contributions_info() diff --git a/backend/stats_app/utils.py b/backend/stats_app/utils.py index 1b9921a4..321b2917 100644 --- a/backend/stats_app/utils.py +++ b/backend/stats_app/utils.py @@ -1,8 +1,14 @@ from datetime import datetime, timedelta from django.utils.timezone import make_aware +from dataclasses import dataclass +from typing import List from base_app.models import User +from backend_app.models.exchangeFeedback import ExchangeFeedback +from backend_app.models.courseFeedback import CourseFeedback +from backend_app.models.exchange import Exchange + def get_today_as_datetime(): now = datetime.now() @@ -17,3 +23,60 @@ def get_daily_connections() -> int: last_login__gte=yesterday, last_login__lt=today ).count() return nb_connections + + +@dataclass(eq=True, frozen=True) +class ContributionProfile: + """ + Class for keeping track a profile. + """ + + major: str + minor: str + exchange_semester: str + university_pk: int + + +def _get_profile_from_exchange(exchange: Exchange) -> ContributionProfile: + return ContributionProfile( + major=exchange.student_major if exchange.student_major is not None else "", + minor=exchange.student_minor if exchange.student_minor is not None else "", + exchange_semester=f"{exchange.semester}{exchange.year}", + university_pk=exchange.university.pk, + ) + + +def get_contributions_profiles() -> List[ContributionProfile]: + """ + return the yesterday contributions profiles + + If no university if associated with a contribution we don't return it + """ + today = get_today_as_datetime() + yesterday = today - timedelta(days=1) + contributions_profiles = [] + + exchange_feedbacks = ExchangeFeedback.objects.filter( + updated_on__gte=yesterday, + updated_on__lt=today, + exchange__university__isnull=False, + untouched=False, + ).prefetch_related("exchange") + + for exchange_feedback in exchange_feedbacks: + exchange = exchange_feedback.exchange + contribution_profile = _get_profile_from_exchange(exchange) + contributions_profiles.append(contribution_profile) + + course_feedbacks = CourseFeedback.objects.filter( + updated_on__gte=yesterday, + updated_on__lt=today, + course__exchange__university__isnull=False, + untouched=False, + ).prefetch_related("course__exchange") + for course_feedback in course_feedbacks: + exchange = course_feedback.course.exchange + contribution_profile = _get_profile_from_exchange(exchange) + contributions_profiles.append(contribution_profile) + + return contributions_profiles -- GitLab From d9920f7ad924dd6cfbdd563c74d9e200e6495cb2 Mon Sep 17 00:00:00 2001 From: Maxime Date: Sun, 7 Jun 2020 11:45:34 +0200 Subject: [PATCH 22/34] test(backend): Add test to check functions of contributions stats computing --- .../stats_app/tests/test_nb_contributions.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 backend/stats_app/tests/test_nb_contributions.py diff --git a/backend/stats_app/tests/test_nb_contributions.py b/backend/stats_app/tests/test_nb_contributions.py new file mode 100644 index 00000000..3c54e286 --- /dev/null +++ b/backend/stats_app/tests/test_nb_contributions.py @@ -0,0 +1,69 @@ +from datetime import timedelta + +from django.test import TestCase + +from backend_app.models.exchangeFeedback import ExchangeFeedback +from backend_app.models.exchange import Exchange +from backend_app.tests.utils import get_dummy_university +from stats_app.compute_stats import update_daily_exchange_contributions_info +from stats_app.models import DailyExchangeContributionsInfo +from stats_app.utils import get_today_as_datetime, get_contributions_profiles + + +class StatsContributionsTest(TestCase): + @classmethod + def setUpTestData(cls): + today = get_today_as_datetime() + yesterday = today - timedelta(days=1) + + cls.univ = get_dummy_university() + utc_partner_id = cls.univ.corresponding_utc_partners.all()[0].utc_id + + for i in range(2): + exchange = Exchange.objects.update_or_create( + pk=i, + defaults=dict( + utc_id=i, + utc_partner_id=utc_partner_id, + year=2019, + semester="A", + student_major_and_semester="IM4", + student_minor="IDI", + duration=2, + double_degree=False, + master_obtained=False, + utc_allow_courses=True, + utc_allow_login=True, + university=cls.univ, + ), + )[0] + ExchangeFeedback.objects.update_or_create( + exchange=exchange, + defaults=dict( + updated_on=yesterday, untouched=False, university=cls.univ + ), + ) + + def test_get_contributions_profiles(self): + daily_contributions_profiles = get_contributions_profiles() + self.assertEqual(len(daily_contributions_profiles), 2) + for p in daily_contributions_profiles: + self.assertEqual(p.major, "IM") + self.assertEqual(p.minor, "IDI") + self.assertEqual(p.exchange_semester, "A2019") + self.assertEqual(p.university_pk, self.univ.pk) + + def test_update_daily_exchange_contributions_info(self): + today = get_today_as_datetime() + yesterday = today - timedelta(days=1) + update_daily_exchange_contributions_info() + + contributions = DailyExchangeContributionsInfo.objects.filter(date=yesterday) + + self.assertEqual(len(contributions), 1) + contribution: DailyExchangeContributionsInfo = contributions[0] + self.assertEqual(contribution.university.pk, self.univ.pk) + self.assertEqual(contribution.major, "IM") + self.assertEqual(contribution.minor, "IDI") + self.assertEqual(contribution.exchange_semester, "A2019") + self.assertEqual(contribution.nb_contributions, 2) -- GitLab From a95d5f686b3966ebd8b7bd042b35cace58e50f85 Mon Sep 17 00:00:00 2001 From: Maxime Date: Sun, 7 Jun 2020 11:47:37 +0200 Subject: [PATCH 23/34] feature(backend): Add function to create dummy partner --- backend/backend_app/tests/utils.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/backend_app/tests/utils.py b/backend/backend_app/tests/utils.py index 5619afe9..240d88d0 100644 --- a/backend/backend_app/tests/utils.py +++ b/backend/backend_app/tests/utils.py @@ -3,6 +3,7 @@ from django.test import TestCase from rest_framework.test import APIClient from backend_app.models.country import Country +from backend_app.models.partner import Partner from backend_app.models.university import University from base_app.models import User @@ -95,7 +96,7 @@ def get_dummy_country() -> Country: def get_dummy_university(id: int = 1) -> University: - return University.objects.update_or_create( + univ = University.objects.update_or_create( name=f"Université de Technologie de Compiègne_{id}", defaults=dict( acronym="UTC", @@ -105,3 +106,16 @@ def get_dummy_university(id: int = 1) -> University: city="", ), )[0] + # Also set up partner info, to complete the circle + partner = get_dummy_partner(id) + partner.university = univ + partner.save() + return univ + + +def get_dummy_partner(id: int = 1) -> Partner: + return Partner.objects.update_or_create( + univ_name=f"Université de Technologie de Compiègne_{id}", + utc_id=id, + defaults=dict(iso_code="ab", country="osef", city=""), + )[0] -- GitLab From cb5dd7960e16a17242f9aecc1b2c76185bfd27be Mon Sep 17 00:00:00 2001 From: Maxime Date: Sun, 7 Jun 2020 11:49:20 +0200 Subject: [PATCH 24/34] refacto(backend): code refactoring to include stats about contributions --- backend/base_app/admin.py | 1 + backend/stats_app/admin.py | 3 ++- backend/stats_app/tests/test_nb_connections.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/base_app/admin.py b/backend/base_app/admin.py index e77eb93d..f085a061 100644 --- a/backend/base_app/admin.py +++ b/backend/base_app/admin.py @@ -15,6 +15,7 @@ class CustomAdminSite(admin.AdminSite): Custom admin site used to add a trigger_cron view on the admin site provided by django """ + def get_urls(self): urls = super().get_urls() urls += [path("trigger_cron/", self.admin_view(trigger_cron))] diff --git a/backend/stats_app/admin.py b/backend/stats_app/admin.py index ae9c8989..b84c48a9 100644 --- a/backend/stats_app/admin.py +++ b/backend/stats_app/admin.py @@ -1,5 +1,6 @@ from base_app.admin import admin_site -from stats_app.models import DailyConnections +from stats_app.models import DailyConnections, DailyExchangeContributionsInfo admin_site.register(DailyConnections) +admin_site.register(DailyExchangeContributionsInfo) diff --git a/backend/stats_app/tests/test_nb_connections.py b/backend/stats_app/tests/test_nb_connections.py index 588664c0..002baa17 100644 --- a/backend/stats_app/tests/test_nb_connections.py +++ b/backend/stats_app/tests/test_nb_connections.py @@ -3,7 +3,7 @@ from datetime import timedelta from django.test import TestCase from base_app.models import User -from stats_app.compute_stats import update_all_stats +from stats_app.compute_stats import update_daily_connections from stats_app.models import DailyConnections from stats_app.utils import get_daily_connections, get_today_as_datetime @@ -29,7 +29,7 @@ class StatsConnectionsTest(TestCase): daily_connections = get_daily_connections() self.assertEqual(daily_connections, 10) - def test_update_all_stats(self): + def test_update_daily_connections(self): today = get_today_as_datetime() yesterday = today - timedelta(days=1) for i in range(10): @@ -37,7 +37,7 @@ class StatsConnectionsTest(TestCase): username=f"{i}", defaults=dict(last_login=yesterday) ) - update_all_stats() + update_daily_connections() yesterday_daily_connections = DailyConnections.objects.get(date=yesterday) -- GitLab From e120ff47d80f56540b791b691c24144dadf9de81 Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Sun, 7 Jun 2020 12:17:22 +0200 Subject: [PATCH 25/34] docs: updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29771bbf..b5a7ca29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [fix] front: univ form was broken (updated backend serializer) - [feat] front / back: added stats view and frontend deps needed for the view - [feat] Create a custom admin site containing a view to run cron tasks manually. +- [feat] back: add functions and tests to compute daily stats about the contributions ## v2.6.0 -- GitLab From 4ca2901724476d30e898e0a7798140b00e9d3aa5 Mon Sep 17 00:00:00 2001 From: Imane Date: Sun, 31 May 2020 11:57:31 +0200 Subject: [PATCH 26/34] feat(PageUniversity) : create lastVisitedUniversity when you click on a new university Relates to #149 --- frontend/src/components/pages/PageUniversity.jsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/pages/PageUniversity.jsx b/frontend/src/components/pages/PageUniversity.jsx index 2a4a0a67..d345c74d 100644 --- a/frontend/src/components/pages/PageUniversity.jsx +++ b/frontend/src/components/pages/PageUniversity.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useEffect } from "react"; import PropTypes from "prop-types"; import Dialog from "@material-ui/core/Dialog"; import DialogActions from "@material-ui/core/DialogActions"; @@ -11,6 +11,7 @@ import { withErrorBoundary } from "../common/ErrorBoundary"; import APP_ROUTES from "../../config/appRoutes"; import CustomNavLink from "../common/CustomNavLink"; import UniversityService from "../../services/data/UniversityService"; +import { useApiCreate } from "../../hooks/wrappers/api"; function UniversityNotFound() { return ( @@ -34,13 +35,25 @@ function UniversityNotFound() { ); } +let lastVisitedUniv = -1; + /** * Component holding the page with the university details */ function PageUniversity({ match }) { const { univId, tabName } = match.params; + const createLastVisited = useApiCreate("lastVisitedUniversities"); + + useEffect(() => { + if (lastVisitedUniv !== univId) + createLastVisited({ university: univId }, () => { + lastVisitedUniv = univId; + }); + }, [univId]); + const parsedUnivId = parseInt(univId, 10); + if (UniversityService.hasUniversity(parsedUnivId)) { return ( -- GitLab From 9ff13c604bcd5851e9ec8281acb35eb3b70d14cc Mon Sep 17 00:00:00 2001 From: Imane Date: Sun, 31 May 2020 12:00:18 +0200 Subject: [PATCH 27/34] feat(LastVisitedUniversity) : add select component for lastVisitedUniversities Relates to #149 --- frontend/src/components/app/App.jsx | 2 + .../components/app/LastVisitedUniversity.jsx | 100 ++++++++++++++++++ .../src/components/common/LicenseNotice.jsx | 12 ++- 3 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 frontend/src/components/app/LastVisitedUniversity.jsx diff --git a/frontend/src/components/app/App.jsx b/frontend/src/components/app/App.jsx index 92006b53..c9f00a62 100644 --- a/frontend/src/components/app/App.jsx +++ b/frontend/src/components/app/App.jsx @@ -6,6 +6,7 @@ import { compose } from "recompose"; import { Route, Switch } from "react-router-dom"; import { withErrorBoundary } from "../common/ErrorBoundary"; +import LastVisitedUniversity from "./LastVisitedUniversity"; import PageMap from "../pages/PageMap"; import PageHome from "../pages/PageHome"; import PageUniversity from "../pages/PageUniversity"; @@ -90,6 +91,7 @@ function App() { + ); diff --git a/frontend/src/components/app/LastVisitedUniversity.jsx b/frontend/src/components/app/LastVisitedUniversity.jsx new file mode 100644 index 00000000..38e16741 --- /dev/null +++ b/frontend/src/components/app/LastVisitedUniversity.jsx @@ -0,0 +1,100 @@ +import React from "react"; +import { compose } from "recompose"; +import { makeStyles } from "@material-ui/styles"; +import InputLabel from "@material-ui/core/InputLabel"; +import FormControl from "@material-ui/core/FormControl"; +import Select from "@material-ui/core/Select"; +import MenuItem from "@material-ui/core/MenuItem"; +import PropTypes from "prop-types"; +import { withRouter } from "react-router-dom"; +import APP_ROUTES from "../../config/appRoutes"; +import withNetworkWrapper, { NetWrapParam } from "../../hoc/withNetworkWrapper"; +import UniversityService from "../../services/data/UniversityService"; +import NavigationService from "../../services/NavigationService"; +import LicenseNotice from "../common/LicenseNotice"; + +const useStyles = makeStyles((theme) => ({ + formControl: { + marginLeft: "auto", + marginRight: theme.spacing(2), + padding: theme.spacing(1), + maxWidth: "80%", + position: "sticky", + bottom: theme.spacing(2), + border: "solid", + borderWidth: 2, + borderRadius: theme.shape.borderRadius, + borderColor: theme.palette.primary.main, + borderStyle: "dashed", + backgroundColor: theme.palette.background.paper, + zIndex: 1000, + }, +})); + +function LastVisitedUniversity({ location, lastVisitedUniversities }) { + const classes = useStyles(); + + if ( + location.pathname === APP_ROUTES.base || + location.pathname === APP_ROUTES.themeSettings || + location.pathname === APP_ROUTES.myExchanges || + location.pathname === APP_ROUTES.editPreviousExchangeWithParams || + location.pathname === APP_ROUTES.userWithParams || + location.pathname === APP_ROUTES.aboutProject || + location.pathname === APP_ROUTES.aboutRgpd || + location.pathname === APP_ROUTES.aboutCgu || + location.pathname === APP_ROUTES.aboutUnlinkedPartners || + location.pathname === APP_ROUTES.logout + ) { + return <>; + } + + const handleChange = (event) => { + const univId = event.target.value; + if (univId !== "") NavigationService.goToUniversity(univId); + }; + + return ( +
+ + + Dernières universités visitées + + + + +
+ ); +} + +LastVisitedUniversity.propTypes = { + location: PropTypes.shape({ + pathname: PropTypes.string.isRequired, + }).isRequired, + lastVisitedUniversities: PropTypes.arrayOf( + PropTypes.shape({ + university: PropTypes.number.isRequired, + ts: PropTypes.string.isRequired, + }).isRequired + ).isRequired, +}; + +export default compose( + withNetworkWrapper([ + new NetWrapParam("lastVisitedUniversities", "all", { + addDataToProp: "lastVisitedUniversities", + }), + ]), + withRouter +)(LastVisitedUniversity); diff --git a/frontend/src/components/common/LicenseNotice.jsx b/frontend/src/components/common/LicenseNotice.jsx index 8d1b0696..b0fc8d94 100644 --- a/frontend/src/components/common/LicenseNotice.jsx +++ b/frontend/src/components/common/LicenseNotice.jsx @@ -21,17 +21,23 @@ function LicenseNotice(props) { return ( <> - Ce contenu est sous license   - {props.variant}. Plus d'informations à ce propos sont disponibles  + Ce contenu est sous license  + {props.variant}.
+ Plus d'informations à ce propos sont disponibles  ici. -
+ {props.spacer &&
} ); } LicenseNotice.propTypes = { variant: PropTypes.oneOf(["REX-DRI—PRIVATE", "REX-DRI—BY"]).isRequired, + spacer: PropTypes.bool, +}; + +LicenseNotice.defaultProps = { + spacer: true, }; export default LicenseNotice; -- GitLab From 7c003df366f0cd28f372128b9f07b02b5deceefe Mon Sep 17 00:00:00 2001 From: Imane Date: Sat, 6 Jun 2020 16:37:31 +0200 Subject: [PATCH 28/34] refacto(LastVisitedUniversity): improve style with sticky position Closes #149 --- frontend/src/components/app/LastVisitedUniversity.jsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/app/LastVisitedUniversity.jsx b/frontend/src/components/app/LastVisitedUniversity.jsx index 38e16741..998db69a 100644 --- a/frontend/src/components/app/LastVisitedUniversity.jsx +++ b/frontend/src/components/app/LastVisitedUniversity.jsx @@ -5,6 +5,7 @@ import InputLabel from "@material-ui/core/InputLabel"; import FormControl from "@material-ui/core/FormControl"; import Select from "@material-ui/core/Select"; import MenuItem from "@material-ui/core/MenuItem"; +import Box from "@material-ui/core/Box"; import PropTypes from "prop-types"; import { withRouter } from "react-router-dom"; import APP_ROUTES from "../../config/appRoutes"; @@ -35,7 +36,6 @@ function LastVisitedUniversity({ location, lastVisitedUniversities }) { const classes = useStyles(); if ( - location.pathname === APP_ROUTES.base || location.pathname === APP_ROUTES.themeSettings || location.pathname === APP_ROUTES.myExchanges || location.pathname === APP_ROUTES.editPreviousExchangeWithParams || @@ -55,9 +55,9 @@ function LastVisitedUniversity({ location, lastVisitedUniversities }) { }; return ( -
- - + + + Dernières universités visitées