feat(production backend dockerfile) & enhanced(backend deps) & fix(userInfo bugs)

* Dropped the use of Pandas & updated loading scripts accordingly
* Separated python requirements files
* Updated Dockerfile to be able to also build a production ready image (without dev dependencies)
* Backend images size cut in more than half 🎉
* Updated a bit the documentation related to Docker
* CI now depends on clear image tags
* Fixed the serializers of User and enhanced frontend of userinfo
* Fixed wrong compose in frontend

Fixes #108 #109 #104
Linked to #66 for new prod dockerfile

.gitlab-ci.yml

@@ -17,7 +17,7 @@ stages:
 check_back:
   <<: *only-default
   stage: check
-  image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
+  image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
   before_script:
     - make setup
   script:
@@ -33,7 +33,7 @@ check_back:
 check_front:
   <<: *only-default
   stage: check
-  image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
+  image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
   before_script:
     - cd frontend && cp -R /usr/src/deps/node_modules .
   script:
@@ -44,7 +44,7 @@ check_front:
 test_back:
   <<: *only-default
   stage: test
-  image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
+  image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
   variables:
     POSTGRES_DB: postgres
     POSTGRES_USER: postgres
@@ -68,7 +68,7 @@ test_back:
 test_frontend:
   <<: *only-default
   stage: test
-  image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
+  image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
   before_script:
     - cd frontend && cp -R /usr/src/deps/node_modules .
   script:
@@ -79,7 +79,7 @@ test_frontend:
 flake8:
   <<: *only-default
   stage: lint
-  image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
+  image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
   script:
     - cd backend && flake8
   tags:
@@ -88,7 +88,7 @@ flake8:
 eslint:
   <<: *only-default
   stage: lint
-  image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
+  image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
   before_script:
     - cd frontend && cp -R /usr/src/deps/node_modules .
   script:

backend/Dockerfile

 # This image is based on a python image.
 
-# Use of stretch instead of Alpine for faster install of python packages (especially pandas)
+# Use of stretch instead of Alpine for faster install of python packages
 # Overall performance might be slightly better dut to the use of different lib (but with bigger image size obviously)
 FROM python:3.7.2-slim-stretch
+SHELL ["/bin/bash", "-c"]
 
-# set work directory
 WORKDIR /usr/src/app
 
-# server dependencies
-# python3-dev, libpq-dev and gcc  is for psycopg2-binary
-RUN apt-get update && apt-get install -y --no-install-recommends \
-		libpq-dev \
-		python3-dev \
-		gcc \
+# Installing main python packages
+COPY requirements.txt /usr/src/app/requirements.txt
+RUN pip install --upgrade pip
+
+# python3-dev, libpq-dev and gcc  is for psycopg2-binary and uwsgi
+# We do a lot of && to keep the image size small :)
+RUN BUILD_DEPENCIES='libpq-dev python3-dev gcc' \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+		${BUILD_DEPENCIES} \
 		make \
-	&& rm -rf /var/lib/apt/lists/*
+	&& pip install -r requirements.txt \
+	&& apt-get remove --auto-remove -y ${BUILD_DEPENCIES} \
+	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+# More python dependencies if in dev env
+ARG BUILD_PRODUCTION_IMAGE="false"
 
-# python dependencies
-RUN pip install --upgrade pip
-COPY requirements.txt /usr/src/app/requirements.txt
-RUN pip install -r requirements.txt
+COPY requirements.dev.txt /usr/src/app/requirements.dev.txt
+RUN if [ "x$BUILD_PRODUCTION_IMAGE" = "xfalse" ]; then echo "building image in dev setting" && pip install -r requirements.dev.txt; else echo "building image in production setting"; fi
 
 ## Add the wait script to the image to wait for the database to be up for sure
 ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait

backend/backend_app/load_data/assets/destinations_extracted.csv

-"utc_id","university","city","country","lat","lon","acronyme","website","logo"
+"utc_id","university","city","country","lat","lon","acronym","website","logo"
 1,"Technische Universitat Ilmenau","Ilmenau","DE","50.68386845","10.9329051768032",,,
 2,"Universidad Del Salvador","Buenos Aires","AR","-34.5562653","-58.730053914754",,,
 3,"Swinburne University Of Technology","Victoria","AU","-37.85240845","144.99182255",,,

backend/backend_app/load_data/loading_scripts/loadCountries.py

 from os.path import abspath, join
 
-import pandas as pd
-from backend_app.load_data.shared import ASSETS_PATH
+from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
 from backend_app.models.country import Country
 from base_app.models import User
-
 from .loadGeneric import LoadGeneric
 
 
@@ -16,40 +14,44 @@ class LoadCountries(LoadGeneric):
     def __init__(self, admin: User):
         self.admin = admin
 
-    def load(self):
+    @staticmethod
+    def get_countries_data():
         country_file_loc = abspath(join(ASSETS_PATH, "country.csv"))
-        conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv"))
+        return csv_2_dict_list(country_file_loc)
 
-        country_pd = pd.read_csv(
-            country_file_loc, sep=",", header=0, dtype=object
-        ).fillna("")
+    @staticmethod
+    def get_iso_conv():
+        conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv"))
+        return csv_2_dict_list(conv_alpha_file_loc)
 
+    def load(self):
         # Need to load the information for converting
-        # Countries alpha-3 code to alpha-2 code
-        data_conv = pd.read_csv(conv_alpha_file_loc, sep=",", header=0, na_filter=False)
+        # countries alpha-3 code to alpha-2 code
         conv_alpha = {}
-        for index, row in data_conv.iterrows():
+        for row in self.get_iso_conv():
             conv_alpha[row["alpha-3"]] = row["alpha-2"]
 
-        for index, r in country_pd.iterrows():
-            Iso_3 = str(r["ISO-alpha3 Code"])
-            code_alpha_2 = None
-            if Iso_3 in conv_alpha.keys():
-                code_alpha_2 = conv_alpha[Iso_3]
+        for row in self.get_countries_data():
+            code_iso_3 = str(row["ISO-alpha3 Code"])
+            if code_iso_3 in conv_alpha.keys():
+                code_alpha_2 = conv_alpha[code_iso_3]
             else:
-                print("ignoring country :", Iso_3)
+                print(
+                    "This country is not correctly identified and won't be inserted in db:",
+                    row,
+                )
                 continue
 
             country = Country(
-                name=r["Country or Area"],
+                name=row["Country or Area"],
                 iso_alpha2_code=code_alpha_2,
-                iso_alpha3_code=Iso_3,
-                region_name=r["Region Name"],
-                region_un_code=r["Region Code"],
-                sub_region_name=r["Sub-region Name"],
-                sub_region_un_code=r["Sub-region Code"],
-                intermediate_region_name=r["Intermediate Region Name"],
-                intermediate_region_un_code=r["Intermediate Region Code"],
+                iso_alpha3_code=code_iso_3,
+                region_name=row["Region Name"],
+                region_un_code=row["Region Code"],
+                sub_region_name=row["Sub-region Name"],
+                sub_region_un_code=row["Sub-region Code"],
+                intermediate_region_name=row["Intermediate Region Name"],
+                intermediate_region_un_code=row["Intermediate Region Code"],
             )
             country.save()
             self.add_info_and_save(country, self.admin)

backend/backend_app/load_data/loading_scripts/loadCurrencies.py

@@ -2,7 +2,7 @@ import csv
 from decimal import Decimal
 from os.path import abspath, join
 
-from backend_app.load_data.shared import ASSETS_PATH
+from backend_app.load_data.utils import ASSETS_PATH
 from backend_app.models.currency import Currency
 from base_app.models import User
 

backend/backend_app/load_data/loading_scripts/loadUniversities.py

 from os.path import abspath, join
 
-import pandas as pd
-from backend_app.load_data.shared import ASSETS_PATH
+from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
 from backend_app.models.campus import Campus
 from backend_app.models.city import City
 from backend_app.models.country import Country
 from backend_app.models.university import University
 from base_app.models import User
-
 from .loadGeneric import LoadGeneric
 
 
@@ -19,32 +17,28 @@ class LoadUniversities(LoadGeneric):
     def __init__(self, admin: User):
         self.admin = admin
 
-    def load(self):
+    @staticmethod
+    def get_destination_data():
         destinations_path = abspath(join(ASSETS_PATH, "destinations_extracted.csv"))
+        return csv_2_dict_list(destinations_path)
 
-        data = pd.read_csv(destinations_path, sep=",", header=0, dtype=object).fillna(
-            ""
-        )
-
-        for index, row in data.iterrows():
-            utc_id, univ_name, city_name, country_code, lat, lon, acronym, website, logo = (
-                row
-            )
-            lat = round(float(lat), 6)
-            lon = round(float(lon), 6)
+    def load(self):
+        for row in self.get_destination_data():
+            lat = round(float(row["lat"]), 6)
+            lon = round(float(row["lon"]), 6)
 
-            country = Country.objects.get(pk=country_code)
-            city = City(name=city_name, country=country)
+            country = Country.objects.get(pk=row["country"])
+            city = City(name=row["city"], country=country)
             city.save()
             self.add_info_and_save(city, self.admin)
 
             univ = University.objects.update_or_create(
-                utc_id=utc_id,
+                utc_id=row["utc_id"],
                 defaults={
-                    "name": univ_name,
-                    "acronym": acronym,
-                    "website": website,
-                    "logo": logo,
+                    "name": row["university"],
+                    "acronym": row["acronym"],
+                    "website": row["website"],
+                    "logo": row["logo"],
                 },
             )[0]
             self.add_info_and_save(univ, self.admin)

backend/backend_app/load_data/shared.py

-from os import path
-
-ASSETS_PATH = path.join(path.realpath(__file__), "../assets/")  # noqa: E402

backend/backend_app/load_data/utils.py

+import csv
+from os import path
+
+ASSETS_PATH = path.join(path.realpath(__file__), "../assets/")  # noqa: E402
+
+
+def csv_2_dict_list(fp: str):
+    """
+    Reads a CSV file (with header row!) and returns it as a list of OrderedDict
+    :param fp: csv file path
+    :type fp: str
+    :return:
+    """
+    with open(fp, "r") as f:
+        reader = csv.DictReader(f, delimiter=",")
+        return [r for r in reader]

backend/base_app/models.py

@@ -4,7 +4,6 @@ from django.contrib.auth.models import AbstractUser
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils.functional import cached_property
-from rest_framework import serializers
 from rest_framework.response import Response
 
 from backend_app.models.abstract.base import BaseModelSerializer
@@ -54,17 +53,6 @@ class User(AbstractUser):
 
 
 class UserSerializer(BaseModelSerializer):
-    # Read only fields
-    username = serializers.CharField(read_only=True)
-    first_name = serializers.CharField(read_only=True)
-    last_name = serializers.CharField(read_only=True)
-    email = serializers.EmailField(read_only=True)
-
-    # fields that the user can modify
-    allow_sharing_personal_info = serializers.BooleanField()
-    secondary_email = serializers.EmailField()
-    pseudo = serializers.CharField()
-
     def validate(self, attrs):
         """
         Also validate at the serializer level to prevent error 500
@@ -86,6 +74,7 @@ class UserSerializer(BaseModelSerializer):
             "pseudo",
             "is_staff",
         )
+        read_only_fields = ("username", "first_name", "last_name", "email")
 
 
 class UserViewset(BaseModelViewSet):

backend/requirements.dev.txt

+black==18.9b0
+flake8==3.7.6
+flake8-todo==0.7 # Also lint TODO notes in python
+django-debug-toolbar==1.11
+django-extensions==2.1.5
+pytest-django==3.4.7
+pytest-cov==2.6.1
+pytest-xdist==1.23.0
+pytest-dotenv==0.4.0

backend/requirements.txt

-# REX-DRI
 Django==2.1.7
 psycopg2-binary==2.7.7
 django-cas-ng==3.6.0
@@ -7,20 +6,9 @@ django-filter==2.1.0  # easy filtering on API
 coreapi==2.3.3  # Automatic API doc generation
 django-reversion==3.0.3
 django-reversion-compare==0.8.6
-pytest-django==3.4.7
-pytest-cov==2.6.1
-pytest-xdist==1.23.0
-django-debug-toolbar==1.11
-pandas==0.24.1
 pyyaml==3.13
-django-extensions==2.1.5
 uwsgi==2.0.18
 dotmap==1.3.4
 django-webpack-loader==0.6.0
-pytest-dotenv==0.4.0
-
-# Direct dev depencies
+python-dotenv==0.10.1
 ipython==7.3.0  # For a better Django shell
-black==18.9b0
-flake8==3.7.6
-flake8-todo==0.7 # Also lint TODO notes in python

docker-compose.yml

@@ -13,7 +13,7 @@ services:
   # Service for the backend app.
   backend:
     # Get the image from the registry
-    image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.1.3
+    image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
     # To use a locally built one, comment above, uncomment bellow.
     # build: ./backend
     restart: on-failure

documentation/Technologies/docker.md

@@ -9,9 +9,10 @@ Then the `docker-compose.yml` file coordinates everything: environment variables
 
 You can have a look at those files for more comments.
 
-## Use in Gitlab
+## Use-case
+
+The `Dockerfile` for the `backend` leads to 2 images (one for a dev environment and one for a production environment). The `Dockerfile` leads to 1 image (containing the node_modules for reproducible builds). All of those images are hosted on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in development, CI and production.
 
-Both images for the `backend` and the `frontend` are stored on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in `Gitlab CI` and during development.
 
 ## Updating the images on Gitlab
 
@@ -26,13 +27,13 @@ docker login registry.gitlab.utc.fr
 To update the images stored on the Gitlab repository, run for instance:
 
 ```bash
-docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
+docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend-dev:v0.0.0
 ```
 
 And push it:
 
 ```bash
-docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
+docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.0.0
 ```
 
 :warning: images should be versioned with meaningful tags.
@@ -43,13 +44,20 @@ You can find on [this repo](https://gitlab.utc.fr/rex-dri/ansible/tree/master/bu
 
 ## Updating the images from Gitlab
 
-To do so, run `make docker-pull`. (this should be automatic on image tag version change)
+To do so, run `docker-compose pull`. (this should be automatic on image tag version change)
+
 
+## Troubleshooting
 
-## Issues with `__pycache__`
+### Issues with `__pycache__`
 
-If you have issues at some point with `__pycache__` files, you can delete them:
+If you have issues at some point with `__pycache__` files (or other files generated at runtime), you can :
+* Re-own the repo folder (you don't have the right to modify the files generated from whithin a docker container):
+```bash
+chown -R ${whoami}:${whoami} .
+```
 
+* or delete them:
 ```bash
 find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf
 ```

documentation/docker.md

-Use of Docker
-==============
-
-## General comment
-
-As said in the introduction, this project makes use of `docker` and `docker-compose`. Their are several `Dockerfile` spraid across the project that creates the required `docker` image.
-
-Then the `docker-compose.yml` file coordinates everything: environment variables, ports, volumes, etc.
-
-You can have a look at those files for more comments.
-
-## Use in Gitlab
-
-The `backend` image is also stored on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in `Gitlab CI`.
-
-As it seemed not possible to do the same for the `frontend` image due to the fact that the `node_modules` folder couldn't be kept during `CI`, the image is basically regenerated in `CI`.
-When ran locally, the `npm i` command (installing the `Node` dependencies) is run everytime with `make up` to make sure your `node_modules` folder is up to date.
-
-## Updating the image on Gitlab
-
-If you are not connected to the registry yet, do it:
-
-```bash
-docker login registry.gitlab.utc.fr
-```
-
-To update the images stored on the Gitlab repository, run for instance:
-
-```bash
-docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
-```
-
-And push it:
-
-```bash
-docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
-```
-
-## Updating the images from Gitlab
-
-To do so, run `make docker-pull`.
-
-
-## Issues with `__pycache__`
-
-If you have issues at some point with `__pycache__` files, you can delete them:
-
-```bash
-find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf
-```
-
-## Issues with Django migrations
-
-With the docker setup, Django migrations are created from within the container and as a result those files are own by the docker user.
-
-If you have issues with git regarding the migrations files (i.e. they don't disappear when you change branch), you need to become an owner of the files: `chown -R $(id -u):$(id -g) backend`.

frontend/src/components/pages/PageUser.js

@@ -8,8 +8,8 @@ import CheckCircleIcon from "@material-ui/icons/CheckCircle";
 import Fab from "@material-ui/core/Fab";
 import Grid from "@material-ui/core/Grid";
 import Divider from "@material-ui/core/Divider";
-import {compose} from "redux";
 import UserInfo from "../user/UserInfo";
+import {compose} from "recompose";
 
 
 /**

frontend/src/components/user/UserInfo.js

@@ -7,10 +7,10 @@ import SendIcon from "@material-ui/icons/Send";
 import CancelIcon from "@material-ui/icons/Cancel";
 import CheckCircleIcon from "@material-ui/icons/CheckCircle";
 import CustomComponentForAPI from "../common/CustomComponentForAPI";
-import {compose} from "redux";
 import getActions from "../../redux/api/getActions";
 import {connect} from "react-redux";
 import SameLine from "../common/SameLine";
+import {compose} from "recompose";
 import UserInfoEditor from "./UserInfoEditor";
 import CreateIcon from "@material-ui/icons/Create";
 
@@ -138,18 +138,24 @@ class UserInfo extends CustomComponentForAPI {
                             variant={"h6"}
                             text={"Autre adresse de contact"}/>
         {
-          displayData ?
-            <Button variant="outlined"
-                    color="secondary"
-                    className={classes.button}
-                    href={`mailto:${secondaryEmail}`}>
-              {secondaryEmail} <SendIcon className={classes.rightIcon}/>
-            </Button>
-            : <></>
+          displayData && secondaryEmail !== null ?
+            <>
+              <Button variant="outlined"
+                      color="secondary"
+                      className={classes.button}
+                      href={`mailto:${secondaryEmail}`}>
+                {secondaryEmail} <SendIcon className={classes.rightIcon}/>
+              </Button>
+              <Typography variant="caption">
+                Cette adresse mail a été mise à disposition par la personne comme autre point de contact.
+              </Typography>
+            </>
+            :
+            <Typography variant="caption">
+              Aucune addresse email secondaire n'est disponible.
+            </Typography>
         }
-        <Typography variant="caption">
-          Cette adresse mail a été mise à disposition par la personne comme autre point de contact.
-        </Typography>
+
 
         <div className={classes.spacer}/>