Commit 6e2273e3 authored by Florent Chehab's avatar Florent Chehab

Feature(deploy config) & tweaks

* Deployment is now fully automated with docker / docker-compose (Backend served through uWSGI with Nginx)
* Logs are configured in django and handled by a dedicated docker service (and kept for 30 days)
* Frontend crash logging handled through the backend
* Quick documentation of the deploy setup

Fixes #110 Fixes #66

Also:

* Moved the envs directory to the more general server dir
* New Picture and File model/serializer/viewset added
* Image validator added (didn't use django image field as it wasn't supporting svg)
* Removed symbolink of assets from the frontend in the backend to make sure we can boot the server in no time
parent d1e7419f
Pipeline #39549 passed with stages
in 3 minutes and 16 seconds
......@@ -19,6 +19,7 @@ check_back:
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.1
before_script:
- sh ./backend/init_logs.sh
- make setup
script:
- cd backend && ./manage.py check
......@@ -33,7 +34,7 @@ check_back:
check_front:
<<: *only-default
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.2.0
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......@@ -54,6 +55,7 @@ test_back:
services:
- postgres:10.5
before_script:
- sh ./backend/init_logs.sh
- make setup
script:
- cd backend
......@@ -68,7 +70,7 @@ test_back:
test_frontend:
<<: *only-default
stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.2.0
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......@@ -88,7 +90,7 @@ flake8:
eslint:
<<: *only-default
stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.1.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:v0.2.0
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......
.PHONY: documentation
setup:
bash envs/init.sh
bash server/envs/init.sh
clear_setup:
rm envs/db.env
rm envs/django.env
rm server/envs/db.env
rm server/envs/django.env
up: setup
docker-compose up
docker-compose up --build
dev: up
down_dev:
docker-compose down
init_dev_data:
docker-compose exec backend sh -c "cd backend && ./manage.py shell < init_dev_data.py"
docker-pull:
docker-compose pull
up--build:
docker-compose up --build
reformat_backend:
docker-compose exec backend sh -c "cd backend && black ."
......@@ -53,3 +58,19 @@ documentation:
documentation_clean:
docker-compose exec backend bash -c "cd documentation && make clean"
prod_yml = -f ./server/docker-compose.prod.yml
prod: setup
$(info In production, we need to reset the webpack-stats.json file to make sure the front is up-to-date)
sudo rm -f frontend/webpack-stats.json
docker-compose $(prod_yml) up --build -d
down_prod:
docker-compose $(prod_yml) down
shell_prod_logs:
docker-compose $(prod_yml) exec logs_rotation /bin/sh -c "cd /var/log && /bin/sh"
shell_backend_prod:
docker-compose $(prod_yml) exec backend sh -c "cd backend && bash"
/static
\ No newline at end of file
/static
/media
......@@ -7,9 +7,9 @@ SHELL ["/bin/bash", "-c"]
WORKDIR /usr/src/app
RUN pip install --upgrade pip
# Installing main python packages
COPY requirements.txt /usr/src/app/requirements.txt
RUN pip install --upgrade pip
# python3-dev, libpq-dev and gcc is for psycopg2-binary and uwsgi
# We do a lot of && to keep the image size small :)
......
......@@ -7,61 +7,76 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('backend_app', '0005_auto_20190423_2027'),
]
dependencies = [("backend_app", "0005_auto_20190423_2027")]
operations = [
migrations.RemoveField(
model_name='coursefeedback',
name='is_psf_credit',
),
migrations.RemoveField(
model_name='coursefeedback',
name='university',
),
migrations.RemoveField(
model_name='exchange',
name='courses',
),
migrations.RemoveField(model_name="coursefeedback", name="is_psf_credit"),
migrations.RemoveField(model_name="coursefeedback", name="university"),
migrations.RemoveField(model_name="exchange", name="courses"),
migrations.AddField(
model_name='course',
name='exchange',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='exchange_courses', to='backend_app.Exchange'),
model_name="course",
name="exchange",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="exchange_courses",
to="backend_app.Exchange",
),
),
migrations.AlterField(
model_name='course',
name='link',
model_name="course",
name="link",
field=models.URLField(blank=True, max_length=500, null=True),
),
migrations.AlterField(
model_name='course',
name='nb_credit',
model_name="course",
name="nb_credit",
field=models.PositiveIntegerField(default=0),
),
migrations.AlterField(
model_name='course',
name='utc_exchange_id',
field=models.IntegerField(),
model_name="course", name="utc_exchange_id", field=models.IntegerField()
),
migrations.AlterField(
model_name='coursefeedback',
name='adequation',
field=models.IntegerField(default=0, validators=[django.core.validators.MaxValueValidator(5), django.core.validators.MinValueValidator(-5)]),
model_name="coursefeedback",
name="adequation",
field=models.IntegerField(
default=0,
validators=[
django.core.validators.MaxValueValidator(5),
django.core.validators.MinValueValidator(-5),
],
),
),
migrations.AlterField(
model_name='coursefeedback',
name='course',
field=models.OneToOneField(default=0, on_delete=django.db.models.deletion.CASCADE, related_name='course_feedback', to='backend_app.Course'),
model_name="coursefeedback",
name="course",
field=models.OneToOneField(
default=0,
on_delete=django.db.models.deletion.CASCADE,
related_name="course_feedback",
to="backend_app.Course",
),
),
migrations.AlterField(
model_name='coursefeedback',
name='language_following_ease',
field=models.IntegerField(default=0, validators=[django.core.validators.MaxValueValidator(5), django.core.validators.MinValueValidator(-5)]),
model_name="coursefeedback",
name="language_following_ease",
field=models.IntegerField(
default=0,
validators=[
django.core.validators.MaxValueValidator(5),
django.core.validators.MinValueValidator(-5),
],
),
),
migrations.AlterField(
model_name='coursefeedback',
name='working_dose',
field=models.IntegerField(default=0, validators=[django.core.validators.MaxValueValidator(5), django.core.validators.MinValueValidator(-5)]),
model_name="coursefeedback",
name="working_dose",
field=models.IntegerField(
default=0,
validators=[
django.core.validators.MaxValueValidator(5),
django.core.validators.MinValueValidator(-5),
],
),
),
]
# Generated by Django 2.1.7 on 2019-05-08 08:39
import backend_app.validation.validators
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("backend_app", "0006_auto_20190504_1752"),
]
operations = [
migrations.CreateModel(
name="File",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"file",
models.FileField(
blank=True, null=True, upload_to="files/%Y/%m/%d/"
),
),
(
"title",
models.CharField(blank=True, default="", max_length=200, null=True),
),
(
"licence",
models.CharField(blank=True, default="", max_length=100, null=True),
),
(
"description",
models.CharField(blank=True, default="", max_length=500, null=True),
),
(
"owner",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to=settings.AUTH_USER_MODEL,
),
),
],
options={"abstract": False},
),
migrations.CreateModel(
name="Picture",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"title",
models.CharField(blank=True, default="", max_length=200, null=True),
),
(
"licence",
models.CharField(blank=True, default="", max_length=100, null=True),
),
(
"description",
models.CharField(blank=True, default="", max_length=500, null=True),
),
(
"file",
models.FileField(
blank=True,
null=True,
upload_to="pictures/%Y/%m/%d/",
validators=[backend_app.validation.validators.ImageValidator()],
),
),
(
"owner",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to=settings.AUTH_USER_MODEL,
),
),
],
options={"abstract": False},
),
]
from django.db import models
from rest_framework import serializers
from rest_framework.response import Response
from backend_app.models.abstract.base import (
BaseModel,
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.permissions.app_permissions import NoDelete, IsStaff, IsOwner, ReadOnly
from backend_app.validation.validators import ImageValidator
from base_app.models import User
#########
# Models
#########
class AbstractFile(BaseModel):
owner = models.ForeignKey(User, on_delete=models.SET_NULL, null=True)
file = models.FileField(upload_to="files/%Y/%m/%d/", blank=True, null=True)
title = models.CharField(max_length=200, default="", blank=True, null=True)
licence = models.CharField(max_length=100, default="", blank=True, null=True)
description = models.CharField(max_length=500, default="", blank=True, null=True)
class Meta:
abstract = True
class File(AbstractFile):
pass
class Picture(AbstractFile):
file = models.FileField(
upload_to="pictures/%Y/%m/%d/",
blank=True,
null=True,
validators=[ImageValidator()],
)
#########
# Serializers
#########
class FileSerializer(BaseModelSerializer):
owner = serializers.StringRelatedField(read_only=True)
def save(self, *args, **kwargs):
instance = super().save(*args, **kwargs)
instance.owner = self.get_user_from_request()
instance.save()
return instance
class Meta:
model = File
fields = BaseModelSerializer.Meta.fields + (
"owner",
"file",
"title",
"licence",
"description",
)
class PictureSerializer(FileSerializer):
class Meta:
model = Picture
fields = FileSerializer.Meta.fields
class FileSerializerFileReadOnly(FileSerializer):
file = serializers.FileField(read_only=True)
class PictureSerializerFileReadOnly(PictureSerializer):
file = serializers.FileField(read_only=True)
#########
# ViewSets
#########
class BaseFileViewSet(BaseModelViewSet):
_serializer_not_read_only = None
_serializer_read_only = None
def get_serializer_class(self):
"""
Custom get serializer to make file field readonly after it has been created
"""
if hasattr(self, "request") and self.request.method == "PUT":
return self._serializer_read_only
else:
return self._serializer_not_read_only
def list(self, request, *args, **kwargs):
# Prevent the querying of all objects.
return Response(list())
permission_classes = (NoDelete | IsStaff, IsOwner | IsStaff | ReadOnly)
class FileViewSet(BaseModelViewSet):
_serializer_not_read_only = PictureSerializer
_serializer_read_only = PictureSerializer
queryset = File.objects.all()
end_point_route = "files"
class PictureViewSet(BaseFileViewSet):
_serializer_not_read_only = PictureSerializer
_serializer_read_only = PictureSerializer
queryset = Picture.objects.all()
end_point_route = "pictures"
from os.path import join
import pytest
from django.core.exceptions import ValidationError
......@@ -8,7 +10,9 @@ from backend_app.validation.validators import (
PhotosValidator,
TaggedItemValidator,
ThemeValidator,
ImageValidator,
)
from base_app.settings.dir_locations import BACKEND_ROOT_DIR
class TestUsefulLinksValidator(object):
......@@ -152,3 +156,45 @@ class TestTagNameValidator(object):
# ie a tag name that doesn't have a matching schema
with pytest.raises(ValidationError):
self.validator("qmlskdjmlqfinzoieuncmlkqsdnkuy")
class TestImageValidator(object):
validator = ImageValidator()
# With file path input
def test_on_valid_png_fp(self):
self.validator(
join(BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon-16x16.png")
)
def test_on_valid_svg_fp(self):
self.validator(
join(BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon.svg")
)
def test_on_invalid_file_fp(self):
with pytest.raises(ValidationError):
self.validator(__file__)
# With file input directly
def test_on_valid_png(self):
with open(
join(
BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon-16x16.png"
),
"rb",
) as f:
self.validator(f)
def test_on_valid_svg(self):
with open(
join(BACKEND_ROOT_DIR, "base_app/static/base_app/favicon/favicon.svg"), "rb"
) as f:
self.validator(f)
def test_on_invalid_file(self):
with open(__file__, "rb") as f:
with pytest.raises(ValidationError):
self.validator(f)
import imghdr
import xml.etree.cElementTree as et
from django.core.exceptions import ValidationError
from django.utils.deconstruct import deconstructible
from jsonschema import FormatChecker
......@@ -150,3 +153,42 @@ class TagNameValidator(object):
tag_name
)
)
@deconstructible()
class ImageValidator(object):
"""
Validator to be check that a file is a valid image.
Can't be tricked, definitely bulletproof.
"""
# Plus svg
allowed_format = ["jpeg", "png", "webp"]
def __call__(self, file):
self.is_image(file)
def is_image(self, fp):
if imghdr.what(fp) not in self.allowed_format:
if type(fp) is str:
with open(fp, "r") as f:
if not self.is_svg(f):
raise ValidationError("Image not recognized")
else:
if not self.is_svg(fp):
raise ValidationError("Image not recognized")
@staticmethod
def is_svg(f):
"""
Check if the provided file is svg
"""
f.seek(0)
tag = None
try:
for event, el in et.iterparse(f, ("start",)):
tag = el.tag
break
except et.ParseError:
pass
return tag == "{http://www.w3.org/2000/svg}svg"
import logging
from django.conf import settings
from rest_framework.response import Response
from rest_framework.views import APIView
......@@ -16,6 +18,7 @@ from backend_app.models.courseFeedback import CourseFeedback
from backend_app.models.currency import CurrencyViewSet
from backend_app.models.department import DepartmentViewSet
from backend_app.models.exchangeFeedback import ExchangeFeedbackViewSet
from backend_app.models.file_picture import FileViewSet, PictureViewSet
from backend_app.models.for_testing.moderation import ForTestingModerationViewSet
from backend_app.models.for_testing.versioning import ForTestingVersioningViewSet
from backend_app.models.offer import OfferViewSet
......@@ -62,6 +65,8 @@ ALL_API_VIEWSETS = [
OfferViewSet,
PendingModerationViewSet,
PendingModerationObjViewSet,
FileViewSet,
PictureViewSet,
ExchangeFeedbackViewSet,
SpecialtyViewSet,
TagViewSet,
......@@ -99,7 +104,25 @@ class AppModerationStatusViewSet(APIView):
)
ALL_API_VIEW_VIEWSETS = [AppModerationStatusViewSet]
class LogFrontendErrorsViewSet(APIView):
"""
Viewset to handle the logging of errors coming from the frontend.
"""
permission_classes = tuple()
end_point_route = "frontendErrors"
def post(self, request):
logger = logging.getLogger("frontend")
data = request.data
if "componentStack" in data.keys():
logger.error(request.data["componentStack"])
else:
logger.error(request.data)
return Response(status=201)
ALL_API_VIEW_VIEWSETS = [AppModerationStatusViewSet, LogFrontendErrorsViewSet]
ALL_VIEWSETS = ALL_API_VIEWSETS + ALL_API_VIEW_VIEWSETS
......
......@@ -9,4 +9,4 @@ from os.path import dirname, join
BACKEND_ROOT_DIR = dirname(dirname(dirname(os.path.abspath(__file__))))
REPO_ROOT_DIR = dirname(BACKEND_ROOT_DIR)
ENVS_FILE_DIR = join(REPO_ROOT_DIR, "envs/")
ENVS_FILE_DIR = join(REPO_ROOT_DIR, "server/envs/")
......@@ -30,7 +30,6 @@ try:
except KeyError:
raise Exception("Env variable missing. Please run `make setup`")
ROOT_URLCONF = "base_app.urls"
WSGI_APPLICATION = "base_app.wsgi.application"
......@@ -144,6 +143,7 @@ if os.environ["ENV"] == "DEV":
DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": "base_app.utils.show_toolbar"}
else:
DEBUG = False
ALLOWED_HOSTS = ["127.0.0.1", "localhost", "rex.dri.utc.fr"]
TESTING = "pytest" in sys.modules
......@@ -257,3 +257,59 @@ TIME_ZONE = "Europe/Paris"
USE_I18N = True
USE_L10N = True
USE_TZ = True
#
#
#####################################
#
# Logging
#
#####################################
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"},
"require_debug_true": {"()": "django.utils.log.RequireDebugTrue"},
},
"formatters": {
"verbose": {
"format": "%(levelname)s|%(asctime)s|%(module)s|%(process)d|%(thread)d|%(message)s",
"datefmt": "%d/%b/%Y %H:%M:%S",
}
},
"handlers": {
"console": {
"level": "INFO",
"filters": ["require_debug_true"],
"class": "logging.StreamHandler",
},
"mail_admins": {
"level": "ERROR",
"filters": ["require_debug_false"],
"class": "django.utils.log.AdminEmailHandler",
},
"log_to_file_django": {
"level": "ERROR",
"filters": ["require_debug_false"],
"class": "logging.FileHandler",
"formatter": "verbose",
"filename": "/var/log/django/error.log",
},
"log_to_file_frontend": {
"level": "ERROR",
"filters": ["require_debug_false"],
"class": "logging.FileHandler",
"formatter": "verbose",
"filename": "/var/log/frontend/error.log",
},
},
"loggers": {
"django": {
"handlers": ["console", "log_to_file_django", "mail_admins"],
"level": "INFO",
},
"frontend": {"handlers": ["console", "log_to_file_frontend"], "level": "INFO"},
},
}
......@@ -33,15 +33,13 @@
</div>
</body>
<link rel="stylesheet" href="{% static '/base_app/frontend_dist/leaflet-dist/leaflet.css' %}"/>
<link rel="stylesheet" href="{% static '/base_app/frontend_dist/custom_leaflet.css' %}"/>
<script>
var __AppUserId = {{ user.pk }};
var __AllBackendEndPointsRoutes = {{ endpoints|safe }};
</script>