Commit 7b30fd5f authored by Florent Chehab's avatar Florent Chehab

feat(production backend dockerfile) & enhanced(backend deps) & fix(userInfo bugs)

* Dropped the use of Pandas & updated loading scripts accordingly
* Separated python requirements files
* Updated Dockerfile to be able to also build a production ready image (without dev dependencies)
* Backend images size cut in more than half 🎉
* Updated a bit the documentation related to Docker
* CI now depends on clear image tags
* Fixed the serializers of User and enhanced frontend of userinfo
* Fixed wrong compose in frontend

Fixes #108 #109 #104
Linked to #66 for new prod dockerfile
parent 4a42eb71
Pipeline #38370 passed with stages
in 3 minutes and 14 seconds
......@@ -17,7 +17,7 @@ stages:
check_back:
<<: *only-default
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
before_script:
- make setup
script:
......@@ -33,7 +33,7 @@ check_back:
check_front:
<<: *only-default
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......@@ -44,7 +44,7 @@ check_front:
test_back:
<<: *only-default
stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
variables:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
......@@ -68,7 +68,7 @@ test_back:
test_frontend:
<<: *only-default
stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......@@ -79,7 +79,7 @@ test_frontend:
flake8:
<<: *only-default
stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
script:
- cd backend && flake8
tags:
......@@ -88,7 +88,7 @@ flake8:
eslint:
<<: *only-default
stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......
# This image is based on a python image.
# Use of stretch instead of Alpine for faster install of python packages (especially pandas)
# Use of stretch instead of Alpine for faster install of python packages
# Overall performance might be slightly better dut to the use of different lib (but with bigger image size obviously)
FROM python:3.7.2-slim-stretch
SHELL ["/bin/bash", "-c"]
# set work directory
WORKDIR /usr/src/app
# server dependencies
# python3-dev, libpq-dev and gcc is for psycopg2-binary
RUN apt-get update && apt-get install -y --no-install-recommends \
libpq-dev \
python3-dev \
gcc \
# Installing main python packages
COPY requirements.txt /usr/src/app/requirements.txt
RUN pip install --upgrade pip
# python3-dev, libpq-dev and gcc is for psycopg2-binary and uwsgi
# We do a lot of && to keep the image size small :)
RUN BUILD_DEPENCIES='libpq-dev python3-dev gcc' \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
${BUILD_DEPENCIES} \
make \
&& rm -rf /var/lib/apt/lists/*
&& pip install -r requirements.txt \
&& apt-get remove --auto-remove -y ${BUILD_DEPENCIES} \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# More python dependencies if in dev env
ARG BUILD_PRODUCTION_IMAGE="false"
# python dependencies
RUN pip install --upgrade pip
COPY requirements.txt /usr/src/app/requirements.txt
RUN pip install -r requirements.txt
COPY requirements.dev.txt /usr/src/app/requirements.dev.txt
RUN if [ "x$BUILD_PRODUCTION_IMAGE" = "xfalse" ]; then echo "building image in dev setting" && pip install -r requirements.dev.txt; else echo "building image in production setting"; fi
## Add the wait script to the image to wait for the database to be up for sure
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
......
"utc_id","university","city","country","lat","lon","acronyme","website","logo"
"utc_id","university","city","country","lat","lon","acronym","website","logo"
1,"Technische Universitat Ilmenau","Ilmenau","DE","50.68386845","10.9329051768032",,,
2,"Universidad Del Salvador","Buenos Aires","AR","-34.5562653","-58.730053914754",,,
3,"Swinburne University Of Technology","Victoria","AU","-37.85240845","144.99182255",,,
......
from os.path import abspath, join
import pandas as pd
from backend_app.load_data.shared import ASSETS_PATH
from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
from backend_app.models.country import Country
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -16,40 +14,44 @@ class LoadCountries(LoadGeneric):
def __init__(self, admin: User):
self.admin = admin
def load(self):
@staticmethod
def get_countries_data():
country_file_loc = abspath(join(ASSETS_PATH, "country.csv"))
conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv"))
return csv_2_dict_list(country_file_loc)
country_pd = pd.read_csv(
country_file_loc, sep=",", header=0, dtype=object
).fillna("")
@staticmethod
def get_iso_conv():
conv_alpha_file_loc = abspath(join(ASSETS_PATH, "alpha-conv-table.csv"))
return csv_2_dict_list(conv_alpha_file_loc)
def load(self):
# Need to load the information for converting
# Countries alpha-3 code to alpha-2 code
data_conv = pd.read_csv(conv_alpha_file_loc, sep=",", header=0, na_filter=False)
# countries alpha-3 code to alpha-2 code
conv_alpha = {}
for index, row in data_conv.iterrows():
for row in self.get_iso_conv():
conv_alpha[row["alpha-3"]] = row["alpha-2"]
for index, r in country_pd.iterrows():
Iso_3 = str(r["ISO-alpha3 Code"])
code_alpha_2 = None
if Iso_3 in conv_alpha.keys():
code_alpha_2 = conv_alpha[Iso_3]
for row in self.get_countries_data():
code_iso_3 = str(row["ISO-alpha3 Code"])
if code_iso_3 in conv_alpha.keys():
code_alpha_2 = conv_alpha[code_iso_3]
else:
print("ignoring country :", Iso_3)
print(
"This country is not correctly identified and won't be inserted in db:",
row,
)
continue
country = Country(
name=r["Country or Area"],
name=row["Country or Area"],
iso_alpha2_code=code_alpha_2,
iso_alpha3_code=Iso_3,
region_name=r["Region Name"],
region_un_code=r["Region Code"],
sub_region_name=r["Sub-region Name"],
sub_region_un_code=r["Sub-region Code"],
intermediate_region_name=r["Intermediate Region Name"],
intermediate_region_un_code=r["Intermediate Region Code"],
iso_alpha3_code=code_iso_3,
region_name=row["Region Name"],
region_un_code=row["Region Code"],
sub_region_name=row["Sub-region Name"],
sub_region_un_code=row["Sub-region Code"],
intermediate_region_name=row["Intermediate Region Name"],
intermediate_region_un_code=row["Intermediate Region Code"],
)
country.save()
self.add_info_and_save(country, self.admin)
......@@ -2,7 +2,7 @@ import csv
from decimal import Decimal
from os.path import abspath, join
from backend_app.load_data.shared import ASSETS_PATH
from backend_app.load_data.utils import ASSETS_PATH
from backend_app.models.currency import Currency
from base_app.models import User
......
from os.path import abspath, join
import pandas as pd
from backend_app.load_data.shared import ASSETS_PATH
from backend_app.load_data.utils import ASSETS_PATH, csv_2_dict_list
from backend_app.models.campus import Campus
from backend_app.models.city import City
from backend_app.models.country import Country
from backend_app.models.university import University
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -19,32 +17,28 @@ class LoadUniversities(LoadGeneric):
def __init__(self, admin: User):
self.admin = admin
def load(self):
@staticmethod
def get_destination_data():
destinations_path = abspath(join(ASSETS_PATH, "destinations_extracted.csv"))
return csv_2_dict_list(destinations_path)
data = pd.read_csv(destinations_path, sep=",", header=0, dtype=object).fillna(
""
)
for index, row in data.iterrows():
utc_id, univ_name, city_name, country_code, lat, lon, acronym, website, logo = (
row
)
lat = round(float(lat), 6)
lon = round(float(lon), 6)
def load(self):
for row in self.get_destination_data():
lat = round(float(row["lat"]), 6)
lon = round(float(row["lon"]), 6)
country = Country.objects.get(pk=country_code)
city = City(name=city_name, country=country)
country = Country.objects.get(pk=row["country"])
city = City(name=row["city"], country=country)
city.save()
self.add_info_and_save(city, self.admin)
univ = University.objects.update_or_create(
utc_id=utc_id,
utc_id=row["utc_id"],
defaults={
"name": univ_name,
"acronym": acronym,
"website": website,
"logo": logo,
"name": row["university"],
"acronym": row["acronym"],
"website": row["website"],
"logo": row["logo"],
},
)[0]
self.add_info_and_save(univ, self.admin)
......
from os import path
ASSETS_PATH = path.join(path.realpath(__file__), "../assets/") # noqa: E402
import csv
from os import path
ASSETS_PATH = path.join(path.realpath(__file__), "../assets/") # noqa: E402
def csv_2_dict_list(fp: str):
"""
Reads a CSV file (with header row!) and returns it as a list of OrderedDict
:param fp: csv file path
:type fp: str
:return:
"""
with open(fp, "r") as f:
reader = csv.DictReader(f, delimiter=",")
return [r for r in reader]
......@@ -4,7 +4,6 @@ from django.contrib.auth.models import AbstractUser
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.functional import cached_property
from rest_framework import serializers
from rest_framework.response import Response
from backend_app.models.abstract.base import BaseModelSerializer
......@@ -54,17 +53,6 @@ class User(AbstractUser):
class UserSerializer(BaseModelSerializer):
# Read only fields
username = serializers.CharField(read_only=True)
first_name = serializers.CharField(read_only=True)
last_name = serializers.CharField(read_only=True)
email = serializers.EmailField(read_only=True)
# fields that the user can modify
allow_sharing_personal_info = serializers.BooleanField()
secondary_email = serializers.EmailField()
pseudo = serializers.CharField()
def validate(self, attrs):
"""
Also validate at the serializer level to prevent error 500
......@@ -86,6 +74,7 @@ class UserSerializer(BaseModelSerializer):
"pseudo",
"is_staff",
)
read_only_fields = ("username", "first_name", "last_name", "email")
class UserViewset(BaseModelViewSet):
......
black==18.9b0
flake8==3.7.6
flake8-todo==0.7 # Also lint TODO notes in python
django-debug-toolbar==1.11
django-extensions==2.1.5
pytest-django==3.4.7
pytest-cov==2.6.1
pytest-xdist==1.23.0
pytest-dotenv==0.4.0
# REX-DRI
Django==2.1.7
psycopg2-binary==2.7.7
django-cas-ng==3.6.0
......@@ -7,20 +6,9 @@ django-filter==2.1.0 # easy filtering on API
coreapi==2.3.3 # Automatic API doc generation
django-reversion==3.0.3
django-reversion-compare==0.8.6
pytest-django==3.4.7
pytest-cov==2.6.1
pytest-xdist==1.23.0
django-debug-toolbar==1.11
pandas==0.24.1
pyyaml==3.13
django-extensions==2.1.5
uwsgi==2.0.18
dotmap==1.3.4
django-webpack-loader==0.6.0
pytest-dotenv==0.4.0
# Direct dev depencies
python-dotenv==0.10.1
ipython==7.3.0 # For a better Django shell
black==18.9b0
flake8==3.7.6
flake8-todo==0.7 # Also lint TODO notes in python
......@@ -13,7 +13,7 @@ services:
# Service for the backend app.
backend:
# Get the image from the registry
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.1.3
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
# To use a locally built one, comment above, uncomment bellow.
# build: ./backend
restart: on-failure
......
......@@ -9,9 +9,10 @@ Then the `docker-compose.yml` file coordinates everything: environment variables
You can have a look at those files for more comments.
## Use in Gitlab
## Use-case
The `Dockerfile` for the `backend` leads to 2 images (one for a dev environment and one for a production environment). The `Dockerfile` leads to 1 image (containing the node_modules for reproducible builds). All of those images are hosted on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in development, CI and production.
Both images for the `backend` and the `frontend` are stored on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in `Gitlab CI` and during development.
## Updating the images on Gitlab
......@@ -26,13 +27,13 @@ docker login registry.gitlab.utc.fr
To update the images stored on the Gitlab repository, run for instance:
```bash
docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend-dev:v0.0.0
```
And push it:
```bash
docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.0.0
```
:warning: images should be versioned with meaningful tags.
......@@ -43,13 +44,20 @@ You can find on [this repo](https://gitlab.utc.fr/rex-dri/ansible/tree/master/bu
## Updating the images from Gitlab
To do so, run `make docker-pull`. (this should be automatic on image tag version change)
To do so, run `docker-compose pull`. (this should be automatic on image tag version change)
## Troubleshooting
## Issues with `__pycache__`
### Issues with `__pycache__`
If you have issues at some point with `__pycache__` files, you can delete them:
If you have issues at some point with `__pycache__` files (or other files generated at runtime), you can :
* Re-own the repo folder (you don't have the right to modify the files generated from whithin a docker container):
```bash
chown -R ${whoami}:${whoami} .
```
* or delete them:
```bash
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf
```
......
Use of Docker
==============
## General comment
As said in the introduction, this project makes use of `docker` and `docker-compose`. Their are several `Dockerfile` spraid across the project that creates the required `docker` image.
Then the `docker-compose.yml` file coordinates everything: environment variables, ports, volumes, etc.
You can have a look at those files for more comments.
## Use in Gitlab
The `backend` image is also stored on [https://gitlab.utc.fr/rex-dri/rex-dri/container_registry](https://gitlab.utc.fr/rex-dri/rex-dri/container_registry) to be easily used in `Gitlab CI`.
As it seemed not possible to do the same for the `frontend` image due to the fact that the `node_modules` folder couldn't be kept during `CI`, the image is basically regenerated in `CI`.
When ran locally, the `npm i` command (installing the `Node` dependencies) is run everytime with `make up` to make sure your `node_modules` folder is up to date.
## Updating the image on Gitlab
If you are not connected to the registry yet, do it:
```bash
docker login registry.gitlab.utc.fr
```
To update the images stored on the Gitlab repository, run for instance:
```bash
docker build ./backend --compress --tag registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
```
And push it:
```bash
docker push registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
```
## Updating the images from Gitlab
To do so, run `make docker-pull`.
## Issues with `__pycache__`
If you have issues at some point with `__pycache__` files, you can delete them:
```bash
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | sudo xargs rm -rf
```
## Issues with Django migrations
With the docker setup, Django migrations are created from within the container and as a result those files are own by the docker user.
If you have issues with git regarding the migrations files (i.e. they don't disappear when you change branch), you need to become an owner of the files: `chown -R $(id -u):$(id -g) backend`.
......@@ -8,8 +8,8 @@ import CheckCircleIcon from "@material-ui/icons/CheckCircle";
import Fab from "@material-ui/core/Fab";
import Grid from "@material-ui/core/Grid";
import Divider from "@material-ui/core/Divider";
import {compose} from "redux";
import UserInfo from "../user/UserInfo";
import {compose} from "recompose";
/**
......
......@@ -7,10 +7,10 @@ import SendIcon from "@material-ui/icons/Send";
import CancelIcon from "@material-ui/icons/Cancel";
import CheckCircleIcon from "@material-ui/icons/CheckCircle";
import CustomComponentForAPI from "../common/CustomComponentForAPI";
import {compose} from "redux";
import getActions from "../../redux/api/getActions";
import {connect} from "react-redux";
import SameLine from "../common/SameLine";
import {compose} from "recompose";
import UserInfoEditor from "./UserInfoEditor";
import CreateIcon from "@material-ui/icons/Create";
......@@ -138,18 +138,24 @@ class UserInfo extends CustomComponentForAPI {
variant={"h6"}
text={"Autre adresse de contact"}/>
{
displayData ?
<Button variant="outlined"
color="secondary"
className={classes.button}
href={`mailto:${secondaryEmail}`}>
{secondaryEmail} <SendIcon className={classes.rightIcon}/>
</Button>
: <></>
displayData && secondaryEmail !== null ?
<>
<Button variant="outlined"
color="secondary"
className={classes.button}
href={`mailto:${secondaryEmail}`}>
{secondaryEmail} <SendIcon className={classes.rightIcon}/>
</Button>
<Typography variant="caption">
Cette adresse mail a été mise à disposition par la personne comme autre point de contact.
</Typography>
</>
:
<Typography variant="caption">
Aucune addresse email secondaire n'est disponible.
</Typography>
}
<Typography variant="caption">
Cette adresse mail a été mise à disposition par la personne comme autre point de contact.
</Typography>
<div className={classes.spacer}/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment