Commit 7b30fd5f authored by Florent Chehab's avatar Florent Chehab

feat(production backend dockerfile) & enhanced(backend deps) & fix(userInfo bugs)

* Dropped the use of Pandas & updated loading scripts accordingly
* Separated python requirements files
* Updated Dockerfile to be able to also build a production ready image (without dev dependencies)
* Backend images size cut in more than half 🎉
* Updated a bit the documentation related to Docker
* CI now depends on clear image tags
* Fixed the serializers of User and enhanced frontend of userinfo
* Fixed wrong compose in frontend

Fixes #108 #109 #104
Linked to #66 for new prod dockerfile
parent 4a42eb71
Pipeline #38370 passed with stages
in 3 minutes and 14 seconds
...@@ -17,7 +17,7 @@ stages: ...@@ -17,7 +17,7 @@ stages:
check_back: check_back:
<<: *only-default <<: *only-default
stage: check stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
before_script: before_script:
- make setup - make setup
script: script:
...@@ -33,7 +33,7 @@ check_back: ...@@ -33,7 +33,7 @@ check_back:
check_front: check_front:
<<: *only-default <<: *only-default
stage: check stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
before_script: before_script:
- cd frontend && cp -R /usr/src/deps/node_modules . - cd frontend && cp -R /usr/src/deps/node_modules .
script: script:
...@@ -44,7 +44,7 @@ check_front: ...@@ -44,7 +44,7 @@ check_front:
test_back: test_back:
<<: *only-default <<: *only-default
stage: test stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
variables: variables:
POSTGRES_DB: postgres POSTGRES_DB: postgres
POSTGRES_USER: postgres POSTGRES_USER: postgres
...@@ -68,7 +68,7 @@ test_back: ...@@ -68,7 +68,7 @@ test_back:
test_frontend: test_frontend:
<<: *only-default <<: *only-default
stage: test stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
before_script: before_script:
- cd frontend && cp -R /usr/src/deps/node_modules . - cd frontend && cp -R /usr/src/deps/node_modules .
script: script:
...@@ -79,7 +79,7 @@ test_frontend: ...@@ -79,7 +79,7 @@ test_frontend:
flake8: flake8:
<<: *only-default <<: *only-default
stage: lint stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
script: script:
- cd backend && flake8 - cd backend && flake8
tags: tags:
...@@ -88,7 +88,7 @@ flake8: ...@@ -88,7 +88,7 @@ flake8:
eslint: eslint:
<<: *only-default <<: *only-default
stage: lint stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
before_script: before_script:
- cd frontend && cp -R /usr/src/deps/node_modules . - cd frontend && cp -R /usr/src/deps/node_modules .
script: script:
......
# This image is based on a python image. # This image is based on a python image.
# Use of stretch instead of Alpine for faster install of python packages (especially pandas) # Use of stretch instead of Alpine for faster install of python packages
# Overall performance might be slightly better dut to the use of different lib (but with bigger image size obviously) # Overall performance might be slightly better dut to the use of different lib (but with bigger image size obviously)
FROM python:3.7.2-slim-stretch FROM python:3.7.2-slim-stretch
SHELL ["/bin/bash", "-c"]
# set work directory
WORKDIR /usr/src/app WORKDIR /usr/src/app
# server dependencies # Installing main python packages
# python3-dev, libpq-dev and gcc is for psycopg2-binary COPY requirements.txt /usr/src/app/requirements.txt
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN pip install --upgrade pip
libpq-dev \
python3-dev \ # python3-dev, libpq-dev and gcc is for psycopg2-binary and uwsgi
gcc \ # We do a lot of && to keep the image size small :)
RUN BUILD_DEPENCIES='libpq-dev python3-dev gcc' \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
${BUILD_DEPENCIES} \
make \ make \
&& rm -rf /var/lib/apt/lists/* && pip install -r requirements.txt \
&& apt-get remove --auto-remove -y ${BUILD_DEPENCIES} \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# More python dependencies if in dev env
ARG BUILD_PRODUCTION_IMAGE="false"
# python dependencies COPY requirements.dev.txt /usr/src/app/requirements.dev.txt
RUN pip install --upgrade pip RUN if [ "x$BUILD_PRODUCTION_IMAGE" = "xfalse" ]; then echo "building image in dev setting" && pip install -r requirements.dev.txt; else echo "building image in production setting"; fi
COPY requirements.txt /usr/src/app/requirements.txt
RUN pip install -r requirements.txt
## Add the wait script to the image to wait for the database to be up for sure ## Add the wait script to the image to wait for the database to be up for sure
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
......
"utc_id","university","city","country","lat","lon","acronyme","website","logo" "utc_id","university","city","country","lat","lon","acronym","website","logo"
1,"Technische Universitat Ilmenau","Ilmenau","DE","50.68386845","10.9329051768032",,, 1,"Technische Universitat Ilmenau","Ilmenau","DE","50.68386845","10.9329051768032",,,
2,"Universidad Del Salvador","Buenos Aires","AR","-34.5562653","-58.730053914754",,, 2,"Universidad Del Salvador","Buenos Aires","AR","-34.5562653","-58.730053914754",,,
3,"Swinburne University Of Technology","Victoria","AU","-37.85240845","144.99182255",,, 3,"Swinburne University Of Technology","Victoria","AU","-37.85240845","144.99182255",,,
......
from os.path import abspath, join from os.path import abspath, join
import pandas as pd from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
from backend_app.load_data.shared import ASSETS_PATH
from backend_app.models.country import Country from backend_app.models.country import Country
from base_app.models import User from base_app.models import User
from .loadGeneric import LoadGeneric from .loadGeneric import LoadGeneric
...@@ -16,40 +14,44 @@ class LoadCountries(LoadGeneric): ...@@ -16,40 +14,44 @@ class LoadCountries(LoadGeneric):
def __init__(self, admin: User): def __init__(self, admin: User):
self.admin = admin self.admin = admin
def load(self): @staticmethod
def get_countries_data():
country_file_loc = abspath(join(ASSETS_PATH, "country.csv")) country_file_loc = abspath(join(ASSETS_PATH, "country.csv"))
conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv")) return csv_2_dict_list(country_file_loc)
country_pd = pd.read_csv( @staticmethod
country_file_loc, sep=",", header=0, dtype=object def get_iso_conv():
).fillna("") conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv"))
return csv_2_dict_list(conv_alpha_file_loc)
def load(self):
# Need to load the information for converting # Need to load the information for converting
# Countries alpha-3 code to alpha-2 code # countries alpha-3 code to alpha-2 code
data_conv = pd.read_csv(conv_alpha_file_loc, sep=",", header=0, na_filter=False)
conv_alpha = {} conv_alpha = {}
for index, row in data_conv.iterrows(): for row in self.get_iso_conv():
conv_alpha[row["alpha-3"]] = row["alpha-2"] conv_alpha[row["alpha-3"]] = row["alpha-2"]
for index, r in country_pd.iterrows(): for row in self.get_countries_data():
Iso_3 = str(r["ISO-alpha3 Code"]) code_iso_3 = str(row["ISO-alpha3 Code"])
code_alpha_2 = None if code_iso_3 in conv_alpha.keys():
if Iso_3 in conv_alpha.keys(): code_alpha_2 = conv_alpha[code_iso_3]
code_alpha_2 = conv_alpha[Iso_3]
else: else:
print("ignoring country :", Iso_3) print(
"This country is not correctly identified and won't be inserted in db:",
row,
)
continue continue
country = Country( country = Country(
name=r["Country or Area"], name=row["Country or Area"],
iso_alpha2_code=code_alpha_2, iso_alpha2_code=code_alpha_2,
iso_alpha3_code=Iso_3, iso_alpha3_code=code_iso_3,
region_name=r["Region Name"], region_name=row["Region Name"],
region_un_code=r["Region Code"], region_un_code=row["Region Code"],
sub_region_name=r["Sub-region Name"], sub_region_name=row["Sub-region Name"],
sub_region_un_code=r["Sub-region Code"], sub_region_un_code=row["Sub-region Code"],
intermediate_region_name=r["Intermediate Region Name"], intermediate_region_name=row["Intermediate Region Name"],
intermediate_region_un_code=r["Intermediate Region Code"], intermediate_region_un_code=row["Intermediate Region Code"],
) )
country.save() country.save()
self.add_info_and_save(country, self.admin) self.add_info_and_save(country, self.admin)
...@@ -2,7 +2,7 @@ import csv ...@@ -2,7 +2,7 @@ import csv
from decimal import Decimal from decimal import Decimal
from os.path import abspath, join from os.path import abspath, join
from backend_app.load_data.shared import ASSETS_PATH from backend_app.load_data.utils import ASSETS_PATH
from backend_app.models.currency import Currency from backend_app.models.currency import Currency
from base_app.models import User from base_app.models import User
......
from os.path import abspath, join from os.path import abspath, join
import pandas as pd from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
from backend_app.load_data.shared import ASSETS_PATH
from backend_app.models.campus import Campus from backend_app.models.campus import Campus
from backend_app.models.city import City from backend_app.models.city import City
from backend_app.models.country import Country from backend_app.models.country import Country
from backend_app.models.university import University from backend_app.models.university import University
from base_app.models import User from base_app.models import User
from .loadGeneric import LoadGeneric from .loadGeneric import LoadGeneric
...@@ -19,32 +17,28 @@ class LoadUniversities(LoadGeneric): ...@@ -19,32 +17,28 @@ class LoadUniversities(LoadGeneric):
def __init__(self, admin: User): def __init__(self, admin: User):
self.admin = admin self.admin = admin
def load(self): @staticmethod
def get_destination_data():
destinations_path = abspath(join(ASSETS_PATH, "destinations_extracted.csv")) destinations_path = abspath(join(ASSETS_PATH, "destinations_extracted.csv"))
return csv_2_dict_list(destinations_path)
data = pd.read_csv(destinations_path, sep=",", header=0, dtype=object).fillna( def load(self):
"" for row in self.get_destination_data():
) lat = round(float(row["lat"]), 6)
lon = round(float(row["lon"]), 6)
for index, row in data.iterrows():
utc_id, univ_name, city_name, country_code, lat, lon, acronym, website, logo = (
row
)
lat = round(float(lat), 6)
lon = round(float(lon), 6)
country = Country.objects.get(pk=country_code) country = Country.objects.get(pk=row["country"])
city = City(name=city_name, country=country) city = City(name=row["city"], country=country)
city.save() city.save()
self.add_info_and_save(city, self.admin) self.add_info_and_save(city, self.admin)
univ = University.objects.update_or_create( univ = University.objects.update_or_create(
utc_id=utc_id, utc_id=row["utc_id"],
defaults={ defaults={
"name": univ_name, "name": row["university"],
"acronym": acronym, "acronym": row["acronym"],
"website": website, "website": row["website"],
"logo": logo, "logo": row["logo"],
}, },
)[0] )[0]
self.add_info_and_save(univ, self.admin) self.add_info_and_save(univ, self.admin)
......
from os import path
ASSETS_PATH = path.join(path.realpath(__file__), "../assets/") # noqa: E402
import csv
from os import path
ASSETS_PATH = path.join(path.realpath(__file__), "../assets/") # noqa: E402
def csv_2_dict_list(fp: str):
"""
Reads a CSV file (with header row!) and returns it as a list of OrderedDict
:param fp: csv file path
:type fp: str
:return:
"""
with open(fp, "r") as f:
reader = csv.DictReader(f, delimiter=",")
return [r for r in reader]
...@@ -4,7 +4,6 @@ from django.contrib.auth.models import AbstractUser ...@@ -4,7 +4,6 @@ from django.contrib.auth.models import AbstractUser
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.db import models from django.db import models
from django.utils.functional import cached_property from django.utils.functional import cached_property
from rest_framework import serializers
from rest_framework.response import Response from rest_framework.response import Response
from backend_app.models.abstract.base import BaseModelSerializer from backend_app.models.abstract.base import BaseModelSerializer
...@@ -54,17 +53,6 @@ class User(AbstractUser): ...@@ -54,17 +53,6 @@ class User(AbstractUser):
class UserSerializer(BaseModelSerializer): class UserSerializer(BaseModelSerializer):
# Read only fields
username = serializers.CharField(read_only=True)
first_name = serializers.CharField(read_only=True)
last_name = serializers.CharField(read_only=True)
email = serializers.EmailField(read_only=True)
# fields that the user can modify
allow_sharing_personal_info = serializers.BooleanField()
secondary_email = serializers.EmailField()
pseudo = serializers.CharField()
def validate(self, attrs): def validate(self, attrs):
""" """
Also validate at the serializer level to prevent error 500 Also validate at the serializer level to prevent error 500
...@@ -86,6 +74,7 @@ class UserSerializer(BaseModelSerializer): ...@@ -86,6 +74,7 @@ class UserSerializer(BaseModelSerializer):
"pseudo", "pseudo",
"is_staff", "is_staff",
) )
read_only_fields = ("username", "first_name", "last_name", "email")
class UserViewset(BaseModelViewSet): class UserViewset(BaseModelViewSet):
......
black==18.9b0
flake8==3.7.6
flake8-todo==0.7 # Also lint TODO notes in python
django-debug-toolbar==1.11
django-extensions==2.1.5
pytest-django==3.4.7
pytest-cov==2.6.1
pytest-xdist==1.23.0
pytest-dotenv==0.4.0
# REX-DRI
Django==2.1.7 Django==2.1.7
psycopg2-binary==2.7.7 psycopg2-binary==2.7.7
django-cas-ng==3.6.0 django-cas-ng==3.6.0
...@@ -7,20 +6,9 @@ django-filter==2.1.0 # easy filtering on API ...@@ -7,20 +6,9 @@ django-filter==2.1.0 # easy filtering on API
coreapi==2.3.3 # Automatic API doc generation coreapi==2.3.3 # Automatic API doc generation
django-reversion==3.0.3 django-reversion==3.0.3
django-reversion-compare==0.8.6 django-reversion-compare==0.8.6
pytest-django==3.4.7
pytest-cov==2.6.1
pytest-xdist==1.23.0
django-debug-toolbar==1.11
pandas==0.24.1
pyyaml==3.13 pyyaml==3.13
django-extensions==2.1.5
uwsgi==2.0.18 uwsgi==2.0.18
dotmap==1.3.4 dotmap==1.3.4
django-webpack-loader==0.6.0 django-webpack-loader==0.6.0
pytest-dotenv==0.4.0 python-dotenv==0.10.1
# Direct dev depencies
ipython==7.3.0 # For a better Django shell ipython==7.3.0 # For a better Django shell
black==18.9b0
flake8==3.7.6
flake8-todo==0.7 # Also lint TODO notes in python
...@@ -13,7 +13,7 @@ services: ...@@ -13,7 +13,7 @@ services:
# Service for the backend app. # Service for the backend app.
backend: backend:
# Get the image from the registry # Get the image from the registry
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.1.3 image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
# To use a locally built one, comment above, uncomment bellow. # To use a locally built one, comment above, uncomment bellow.
# build: ./backend # build: ./backend
restart: on-failure restart: on-failure
......
...@@ -9,9 +9,10 @@ Then the `docker-compose.yml` file coordinates everything: environment variables ...@@ -9,9 +9,10 @@ Then the `docker-compose.yml` file coordinates everything: environment variables
You can have a look at those files for more comments. You can have a look at those files for more comments.
## Use in Gitlab ## Use-case
The `Dockerfile` for the `backend` leads to 2 images (one for a dev environment and one for a production environment). The `Dockerfile` leads to 1 image (containing the node_modules for reproducible builds). All of those images are hosted on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in development, CI and production.
Both images for the `backend` and the `frontend` are stored on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in `Gitlab CI` and during development.
## Updating the images on Gitlab ## Updating the images on Gitlab
...@@ -26,13 +27,13 @@ docker login registry.gitlab.utc.fr ...@@ -26,13 +27,13 @@ docker login registry.gitlab.utc.fr
To update the images stored on the Gitlab repository, run for instance: To update the images stored on the Gitlab repository, run for instance:
```bash ```bash
docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend-dev:v0.0.0
``` ```
And push it: And push it:
```bash ```bash
docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.0.0
``` ```
:warning: images should be versioned with meaningful tags. :warning: images should be versioned with meaningful tags.
...@@ -43,13 +44,20 @@ You can find on [this repo](https://gitlab.utc.fr/rex-dri/ansible/tree/master/bu ...@@ -43,13 +44,20 @@ You can find on [this repo](https://gitlab.utc.fr/rex-dri/ansible/tree/master/bu
## Updating the images from Gitlab ## Updating the images from Gitlab
To do so, run `make docker-pull`. (this should be automatic on image tag version change) To do so, run `docker-compose pull`. (this should be automatic on image tag version change)
## Troubleshooting
## Issues with `__pycache__` ### Issues with `__pycache__`
If you have issues at some point with `__pycache__` files, you can delete them: If you have issues at some point with `__pycache__` files (or other files generated at runtime), you can :
* Re-own the repo folder (you don't have the right to modify the files generated from whithin a docker container):
```bash
chown -R ${whoami}:${whoami} .
```
* or delete them:
```bash ```bash
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf
``` ```
......
Use of Docker
==============
## General comment
As said in the introduction, this project makes use of `docker` and `docker-compose`. Their are several `Dockerfile` spraid across the project that creates the required `docker` image.
Then the `docker-compose.yml` file coordinates everything: environment variables, ports, volumes, etc.
You can have a look at those files for more comments.
## Use in Gitlab
The `backend` image is also stored on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in `Gitlab CI`.
As it seemed not possible to do the same for the `frontend` image due to the fact that the `node_modules` folder couldn't be kept during `CI`, the image is basically regenerated in `CI`.
When ran locally, the `npm i` command (installing the `Node` dependencies) is run everytime with `make up` to make sure your `node_modules` folder is up to date.
## Updating the image on Gitlab
If you are not connected to the registry yet, do it:
```bash
docker login registry.gitlab.utc.fr
```
To update the images stored on the Gitlab repository, run for instance:
```bash
docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
```
And push it:
```bash
docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
```
## Updating the images from Gitlab
To do so, run `make docker-pull`.
## Issues with `__pycache__`
If you have issues at some point with `__pycache__` files, you can delete them:
```bash
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf
```
## Issues with Django migrations
With the docker setup, Django migrations are created from within the container and as a result those files are own by the docker user.
If you have issues with git regarding the migrations files (i.e. they don't disappear when you change branch), you need to become an owner of the files: `chown -R $(id -u):$(id -g) backend`.
...@@ -8,8 +8,8 @@ import CheckCircleIcon from "@material-ui/icons/CheckCircle"; ...@@ -8,8 +8,8 @@ import CheckCircleIcon from "@material-ui/icons/CheckCircle";
import Fab from "@material-ui/core/Fab"; import Fab from "@material-ui/core/Fab";
import Grid from "@material-ui/core/Grid"; import Grid from "@material-ui/core/Grid";
import Divider from "@material-ui/core/Divider"; import Divider from "@material-ui/core/Divider";
import {compose} from "redux";
import UserInfo from "../user/UserInfo"; import UserInfo from "../user/UserInfo";
import {compose} from "recompose";
/** /**
......
...@@ -7,10 +7,10 @@ import SendIcon from "@material-ui/icons/Send"; ...@@ -7,10 +7,10 @@ import SendIcon from "@material-ui/icons/Send";
import CancelIcon from "@material-ui/icons/Cancel"; import CancelIcon from "@material-ui/icons/Cancel";
import CheckCircleIcon from "@material-ui/icons/CheckCircle"; import CheckCircleIcon from "@material-ui/icons/CheckCircle";
import CustomComponentForAPI from "../common/CustomComponentForAPI"; import CustomComponentForAPI from "../common/CustomComponentForAPI";
import {compose} from "redux";
import getActions from "../../redux/api/getActions"; import getActions from "../../redux/api/getActions";
import {connect} from "react-redux"; import {connect} from "react-redux";
import SameLine from "../common/SameLine"; import SameLine from "../common/SameLine";
import {compose} from "recompose";
import UserInfoEditor from "./UserInfoEditor"; import UserInfoEditor from "./UserInfoEditor";
import CreateIcon from "@material-ui/icons/Create"; import CreateIcon from "@material-ui/icons/Create";
...@@ -138,18 +138,24 @@ class UserInfo extends CustomComponentForAPI { ...@@ -138,18 +138,24 @@ class UserInfo extends CustomComponentForAPI {
variant={"h6"} variant={"h6"}
text={"Autre adresse de contact"}/> text={"Autre adresse de contact"}/>
{ {
displayData ? displayData && secondaryEmail !== null ?
<Button variant="outlined" <>
color="secondary" <Button variant="outlined"
className={classes.button} color="secondary"
href={`mailto:${secondaryEmail}`}> className={classes.button}
{secondaryEmail} <SendIcon className={classes.rightIcon}/> href={`mailto:${secondaryEmail}`}>
</Button> {secondaryEmail} <SendIcon className={classes.rightIcon}/>
: <></> </Button>
<Typography variant="caption">
Cette adresse mail a été mise à disposition par la personne comme autre point de contact.
</Typography>
</>
:
<Typography variant="caption">
Aucune addresse email secondaire n'est disponible.
</Typography>
} }
<Typography variant="caption">
Cette adresse mail a été mise à disposition par la personne comme autre point de contact.
</Typography>
<div className={classes.spacer}/> <div className={classes.spacer}/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment