Commit 6e2273e3 authored by Florent Chehab's avatar Florent Chehab

Feature(deploy config) & tweaks

* Deployment is now fully automated with docker / docker-compose (Backend served through uWSGI with Nginx)
* Logs are configured in django and handled by a dedicated docker service (and kept for 30 days)
* Frontend crash logging handled through the backend
* Quick documentation of the deploy setup

Fixes #110 Fixes #66

Also:

* Moved the envs directory to the more general server dir
* New Picture and File model/serializer/viewset added
* Image validator added (didn't use django image field as it wasn't supporting svg)
* Removed symbolink of assets from the frontend in the backend to make sure we can boot the server in no time
parent d1e7419f
...@@ -19,6 +19,7 @@ check_back: ...@@ -19,6 +19,7 @@ check_back:
stage: check stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.1 image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.1
before_script: before_script:
- sh ./backend/init_logs.sh
- make setup - make setup
script: script:
- cd backend && ./manage.py check - cd backend && ./manage.py check
...@@ -33,7 +34,7 @@ check_back: ...@@ -33,7 +34,7 @@ check_back:
check_front: check_front:
<<: *only-default <<: *only-default
stage: check stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0 image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.2.0
before_script: before_script:
- cd frontend && cp -R /usr/src/deps/node_modules . - cd frontend && cp -R /usr/src/deps/node_modules .
script: script:
...@@ -54,6 +55,7 @@ test_back: ...@@ -54,6 +55,7 @@ test_back:
services: services:
- postgres:10.5 - postgres:10.5
before_script: before_script:
- sh ./backend/init_logs.sh
- make setup - make setup
script: script:
- cd backend - cd backend
...@@ -68,7 +70,7 @@ test_back: ...@@ -68,7 +70,7 @@ test_back:
test_frontend: test_frontend:
<<: *only-default <<: *only-default
stage: test stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0 image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.2.0
before_script: before_script:
- cd frontend && cp -R /usr/src/deps/node_modules . - cd frontend && cp -R /usr/src/deps/node_modules .
script: script:
...@@ -88,7 +90,7 @@ flake8: ...@@ -88,7 +90,7 @@ flake8:
eslint: eslint:
<<: *only-default <<: *only-default
stage: lint stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0 image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.2.0
before_script: before_script:
- cd frontend && cp -R /usr/src/deps/node_modules . - cd frontend && cp -R /usr/src/deps/node_modules .
script: script:
......
.PHONY: documentation .PHONY: documentation
setup: setup:
bash envs/init.sh bash server/envs/init.sh
clear_setup: clear_setup:
rm envs/db.env rm server/envs/db.env
rm envs/django.env rm server/envs/django.env
up: setup up: setup
docker-compose up docker-compose up --build
dev: up
down_dev:
docker-compose down
init_dev_data:
docker-compose exec backend sh -c "cd backend && ./manage.py shell < init_dev_data.py"
docker-pull: docker-pull:
docker-compose pull docker-compose pull
up--build:
docker-compose up --build
reformat_backend: reformat_backend:
docker-compose exec backend sh -c "cd backend && black ." docker-compose exec backend sh -c "cd backend && black ."
...@@ -53,3 +58,19 @@ documentation: ...@@ -53,3 +58,19 @@ documentation:
documentation_clean: documentation_clean:
docker-compose exec backend bash -c "cd documentation && make clean" docker-compose exec backend bash -c "cd documentation && make clean"
prod_yml = -f ./server/docker-compose.prod.yml
prod: setup
$(info In production, we need to reset the webpack-stats.json file to make sure the front is up-to-date)
sudo rm -f frontend/webpack-stats.json
docker-compose $(prod_yml) up --build -d
down_prod:
docker-compose $(prod_yml) down
shell_prod_logs:
docker-compose $(prod_yml) exec logs_rotation /bin/sh -c "cd /var/log && /bin/sh"
shell_backend_prod:
docker-compose $(prod_yml) exec backend sh -c "cd backend && bash"
...@@ -7,9 +7,9 @@ SHELL ["/bin/bash", "-c"] ...@@ -7,9 +7,9 @@ SHELL ["/bin/bash", "-c"]
WORKDIR /usr/src/app WORKDIR /usr/src/app
RUN pip install --upgrade pip
# Installing main python packages # Installing main python packages
COPY requirements.txt /usr/src/app/requirements.txt COPY requirements.txt /usr/src/app/requirements.txt
RUN pip install --upgrade pip
# python3-dev, libpq-dev and gcc is for psycopg2-binary and uwsgi # python3-dev, libpq-dev and gcc is for psycopg2-binary and uwsgi
# We do a lot of && to keep the image size small :) # We do a lot of && to keep the image size small :)
......
...@@ -7,61 +7,76 @@ import django.db.models.deletion ...@@ -7,61 +7,76 @@ import django.db.models.deletion
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [("backend_app", "0005_auto_20190423_2027")]
('backend_app', '0005_auto_20190423_2027'),
]
operations = [ operations = [
migrations.RemoveField( migrations.RemoveField(model_name="coursefeedback", name="is_psf_credit"),
model_name='coursefeedback', migrations.RemoveField(model_name="coursefeedback", name="university"),
name='is_psf_credit', migrations.RemoveField(model_name="exchange", name="courses"),
),
migrations.RemoveField(
model_name='coursefeedback',
name='university',
),
migrations.RemoveField(
model_name='exchange',
name='courses',
),
migrations.AddField( migrations.AddField(
model_name='course', model_name="course",
name='exchange', name="exchange",
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='exchange_courses', to='backend_app.Exchange'), field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="exchange_courses",
to="backend_app.Exchange",
),
), ),
migrations.AlterField( migrations.AlterField(
model_name='course', model_name="course",
name='link', name="link",
field=models.URLField(blank=True, max_length=500, null=True), field=models.URLField(blank=True, max_length=500, null=True),
), ),
migrations.AlterField( migrations.AlterField(
model_name='course', model_name="course",
name='nb_credit', name="nb_credit",
field=models.PositiveIntegerField(default=0), field=models.PositiveIntegerField(default=0),
), ),
migrations.AlterField( migrations.AlterField(
model_name='course', model_name="course", name="utc_exchange_id", field=models.IntegerField()
name='utc_exchange_id',
field=models.IntegerField(),
), ),
migrations.AlterField( migrations.AlterField(
model_name='coursefeedback', model_name="coursefeedback",
name='adequation', name="adequation",
field=models.IntegerField(default=0, validators=[django.core.validators.MaxValueValidator(5), django.core.validators.MinValueValidator(-5)]), field=models.IntegerField(
default=0,
validators=[
django.core.validators.MaxValueValidator(5),
django.core.validators.MinValueValidator(-5),
],
),
), ),
migrations.AlterField( migrations.AlterField(
model_name='coursefeedback', model_name="coursefeedback",
name='course', name="course",
field=models.OneToOneField(default=0, on_delete=django.db.models.deletion.CASCADE, related_name='course_feedback', to='backend_app.Course'), field=models.OneToOneField(
default=0,
on_delete=django.db.models.deletion.CASCADE,
related_name="course_feedback",
to="backend_app.Course",
),
), ),
migrations.AlterField( migrations.AlterField(
model_name='coursefeedback', model_name="coursefeedback",
name='language_following_ease', name="language_following_ease",
field=models.IntegerField(default=0, validators=[django.core.validators.MaxValueValidator(5), django.core.validators.MinValueValidator(-5)]), field=models.IntegerField(
default=0,
validators=[
django.core.validators.MaxValueValidator(5),
django.core.validators.MinValueValidator(-5),
],
),
), ),
migrations.AlterField( migrations.AlterField(
model_name='coursefeedback', model_name="coursefeedback",
name='working_dose', name="working_dose",
field=models.IntegerField(default=0, validators=[django.core.validators.MaxValueValidator(5), django.core.validators.MinValueValidator(-5)]), field=models.IntegerField(
default=0,
validators=[
django.core.validators.MaxValueValidator(5),
django.core.validators.MinValueValidator(-5),
],
),
), ),
] ]
# Generated by Django 2.1.7 on 2019-05-08 08:39
import backend_app.validation.validators
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("backend_app", "0006_auto_20190504_1752"),
]
operations = [
migrations.CreateModel(
name="File",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"file",
models.FileField(
blank=True, null=True, upload_to="files/%Y/%m/%d/"
),
),
(
"title",
models.CharField(blank=True, default="", max_length=200, null=True),
),
(
"licence",
models.CharField(blank=True, default="", max_length=100, null=True),
),
(
"description",
models.CharField(blank=True, default="", max_length=500, null=True),
),
(
"owner",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to=settings.AUTH_USER_MODEL,
),
),
],
options={"abstract": False},
),
migrations.CreateModel(
name="Picture",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"title",
models.CharField(blank=True, default="", max_length=200, null=True),
),
(
"licence",
models.CharField(blank=True, default="", max_length=100, null=True),
),
(
"description",
models.CharField(blank=True, default="", max_length=500, null=True),
),
(
"file",
models.FileField(
blank=True,
null=True,
upload_to="pictures/%Y/%m/%d/",
validators=[backend_app.validation.validators.ImageValidator()],
),
),
(
"owner",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to=settings.AUTH_USER_MODEL,
),
),
],
options={"abstract": False},
),
]
from django.db import models
from rest_framework import serializers
from rest_framework.response import Response
from backend_app.models.abstract.base import (
BaseModel,
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.permissions.app_permissions import NoDelete, IsStaff, IsOwner, ReadOnly
from backend_app.validation.validators import ImageValidator
from base_app.models import User
#########
# Models
#########
class AbstractFile(BaseModel):
owner = models.ForeignKey(User, on_delete=models.SET_NULL, null=True)
file = models.FileField(upload_to="files/%Y/%m/%d/", blank=True, null=True)
title = models.CharField(max_length=200, default="", blank=True, null=True)
licence = models.CharField(max_length=100, default="", blank=True, null=True)
description = models.CharField(max_length=500, default="", blank=True, null=True)
class Meta:
abstract = True
class File(AbstractFile):
pass
class Picture(AbstractFile):
file = models.FileField(
upload_to="pictures/%Y/%m/%d/",
blank=True,
null=True,
validators=[ImageValidator()],
)
#########
# Serializers
#########
class FileSerializer(BaseModelSerializer):
owner = serializers.StringRelatedField(read_only=True)
def save(self, *args, **kwargs):
instance = super().save(*args, **kwargs)
instance.owner = self.get_user_from_request()
instance.save()
return instance
class Meta:
model = File
fields = BaseModelSerializer.Meta.fields + (
"owner",
"file",
"title",
"licence",
"description",
)
class PictureSerializer(FileSerializer):
class Meta:
model = Picture
fields = FileSerializer.Meta.fields
class FileSerializerFileReadOnly(FileSerializer):
file = serializers.FileField(read_only=True)
class PictureSerializerFileReadOnly(PictureSerializer):
file = serializers.FileField(read_only=True)
#########
# ViewSets
#########
class BaseFileViewSet(BaseModelViewSet):
_serializer_not_read_only = None
_serializer_read_only = None
def get_serializer_class(self):
"""
Custom get serializer to make file field readonly after it has been created
"""
if hasattr(self, "request") and self.request.method == "PUT":
return self._serializer_read_only
else:
return self._serializer_not_read_only
def list(self, request, *args, **kwargs):
# Prevent the querying of all objects.
return Response(list())
permission_classes = (NoDelete | IsStaff, IsOwner | IsStaff | ReadOnly)
class FileViewSet(BaseModelViewSet):
_serializer_not_read_only = PictureSerializer
_serializer_read_only = PictureSerializer
queryset = File.objects.all()
end_point_route = "files"
class PictureViewSet(BaseFileViewSet):
_serializer_not_read_only = PictureSerializer
_serializer_read_only = PictureSerializer
queryset = Picture.objects.all()
end_point_route = "pictures"
from os.path import join
import pytest import pytest
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
...@@ -8,7 +10,9 @@ from backend_app.validation.validators import ( ...@@ -8,7 +10,9 @@ from backend_app.validation.validators import (
PhotosValidator, PhotosValidator,
TaggedItemValidator, TaggedItemValidator,
ThemeValidator, ThemeValidator,
ImageValidator,
) )
from base_app.settings.dir_locations import BACKEND_ROOT_DIR
class TestUsefulLinksValidator(object): class TestUsefulLinksValidator(object):
...@@ -152,3 +156,45 @@ class TestTagNameValidator(object): ...@@ -152,3 +156,45 @@ class TestTagNameValidator(object):
# ie a tag name that doesn't have a matching schema # ie a tag name that doesn't have a matching schema
with pytest.raises(ValidationError): with pytest.raises(ValidationError):
self.validator("qmlskdjmlqfinzoieuncmlkqsdnkuy") self.validator("qmlskdjmlqfinzoieuncmlkqsdnkuy")
class TestImageValidator(object):
validator = ImageValidator()
# With file path input
def test_on_valid_png_fp(self):
self.validator(
join(BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon-16x16.png")
)
def test_on_valid_svg_fp(self):
self.validator(
join(BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon.svg")
)
def test_on_invalid_file_fp(self):
with pytest.raises(ValidationError):
self.validator(__file__)
# With file input directly
def test_on_valid_png(self):
with open(
join(
BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon-16x16.png"
),
"rb",
) as f:
self.validator(f)
def test_on_valid_svg(self):
with open(
join(BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon.svg"), "rb"
) as f:
self.validator(f)
def test_on_invalid_file(self):
with open(__file__, "rb") as f:
with pytest.raises(ValidationError):
self.validator(f)
import imghdr
import xml.etree.cElementTree as et
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.utils.deconstruct import deconstructible from django.utils.deconstruct import deconstructible
from jsonschema import FormatChecker from jsonschema import FormatChecker
...@@ -150,3 +153,42 @@ class TagNameValidator(object): ...@@ -150,3 +153,42 @@ class TagNameValidator(object):
tag_name tag_name
) )
) )
@deconstructible()
class ImageValidator(object):
"""
Validator to be check that a file is a valid image.
Can't be tricked, definitely bulletproof.
"""
# Plus svg
allowed_format = ["jpeg", "png", "webp"]
def __call__(self, file):
self.is_image(file)
def is_image(self, fp):
if imghdr.what(fp) not in self.allowed_format:
if type(fp) is str:
with open(fp, "r") as f:
if not self.is_svg(f):
raise ValidationError("Image not recognized")
else:
if not self.is_svg(fp):
raise ValidationError("Image not recognized")
@staticmethod
def is_svg(f):
"""
Check if the provided file is svg
"""
f.seek(0)
tag = None
try:
for event, el in et.iterparse(f, ("start",)):
tag = el.tag
break
except et.ParseError:
pass
return tag == "{http://www.w3.org/2000/svg}svg"
import logging
from django.conf import settings from django.conf import settings
from rest_framework.response import Response from rest_framework.response import Response
from rest_framework.views import APIView from rest_framework.views import APIView
...@@ -16,6 +18,7 @@ from backend_app.models.courseFeedback import CourseFeedback ...@@ -16,6 +18,7 @@ from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import CurrencyViewSet from backend_app.models.currency import CurrencyViewSet
from backend_app.models.department import DepartmentViewSet from backend_app.models.department import DepartmentViewSet
from backend_app.models.exchangeFeedback import ExchangeFeedbackViewSet from backend_app.models.exchangeFeedback import ExchangeFeedbackViewSet
from backend_app.models.file_picture import FileViewSet, PictureViewSet
from backend_app.models.for_testing.moderation import ForTestingModerationViewSet from backend_app.models.for_testing.moderation import ForTestingModerationViewSet
from backend_app.models.for_testing.versioning import ForTestingVersioningViewSet from backend_app.models.for_testing.versioning import ForTestingVersioningViewSet
from backend_app.models.offer import OfferViewSet from backend_app.models.offer import OfferViewSet
...@@ -62,6 +65,8 @@ ALL_API_VIEWSETS = [ ...@@ -62,6 +65,8 @@ ALL_API_VIEWSETS = [
OfferViewSet, OfferViewSet,
PendingModerationViewSet, PendingModerationViewSet,
PendingModerationObjViewSet, PendingModerationObjViewSet,
FileViewSet,
PictureViewSet,
ExchangeFeedbackViewSet, ExchangeFeedbackViewSet,
SpecialtyViewSet, SpecialtyViewSet,
TagViewSet, TagViewSet,
...@@ -99,7 +104,25 @@ class AppModerationStatusViewSet(APIView): ...@@ -99,7 +104,25 @@ class AppModerationStatusViewSet(APIView):
) )
ALL_API_VIEW_VIEWSETS = [AppModerationStatusViewSet] class LogFrontendErrorsViewSet(APIView):
"""
Viewset to handle the logging of errors coming from the frontend.
"""
permission_classes = tuple()
end_point_route = "frontendErrors"
def post(self, request):
logger = logging.getLogger("frontend")
data = request.data
if "componentStack" in data.keys():
logger.error(request.data["componentStack"])
else:
logger.error(request.data)
return Response(status=201)
ALL_API_VIEW_VIEWSETS = [AppModerationStatusViewSet, LogFrontendErrorsViewSet]
ALL_VIEWSETS = ALL_API_VIEWSETS + ALL_API_VIEW_VIEWSETS ALL_VIEWSETS = ALL_API_VIEWSETS + ALL_API_VIEW_VIEWSETS
......
...@@ -9,4 +9,4 @@ from os.path import dirname, join ...@@ -9,4 +9,4 @@ from os.path import dirname, join
BACKEND_ROOT_DIR = dirname(dirname(dirname(os.path.abspath(__file__)))) BACKEND_ROOT_DIR = dirname(dirname(dirname(os.path.abspath(__file__))))
REPO_ROOT_DIR = dirname(BACKEND_ROOT_DIR) REPO_ROOT_DIR = dirname(BACKEND_ROOT_DIR)
ENVS_FILE_DIR = join(REPO_ROOT_DIR, "envs/") ENVS_FILE_DIR = join(REPO_ROOT_DIR, "server/envs/")
...@@ -30,7 +30,6 @@ try: ...@@ -30,7 +30,6 @@ try:
except KeyError: except KeyError:
raise Exception("Env variable missing. Please run `make setup`") raise Exception("Env variable missing. Please run `make setup`")
ROOT_URLCONF = "base_app.urls"