Commit b9500eb2 authored by Florent Chehab's avatar Florent Chehab

Refactor(json validation): moved to standard json-schema

* JSON validation in now performed against `json-schemas` which give a standard way of handling this.
* All previous validation moved to new setup (`useful_links` field & `TaggedItem` in particular)
* Tags handling slightly updated (schemas are now hardcoded in the app and not stored in db)
* All new validators are unitested 🎉
* A bit of documentation added

----
* Bumped backend image to version 0.2.1 with new python packages requirements
----

Fixes #112
Mentions #113 #57
parent 7b30fd5f
Pipeline #38480 passed with stages
in 3 minutes and 10 seconds
......@@ -17,7 +17,7 @@ stages:
check_back:
<<: *only-default
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.1
before_script:
- make setup
script:
......@@ -44,7 +44,7 @@ check_front:
test_back:
<<: *only-default
stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.1
variables:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
......@@ -79,7 +79,7 @@ test_frontend:
flake8:
<<: *only-default
stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.2.1
script:
- cd backend && flake8
tags:
......
[
{
"name": "photos",
"config": {
"photos": {
"type": "photos",
"required": true
}
}
},
{
"name": "insurance",
"config": {}
},
{
"name": "transport",
"config": {}
},
{
"name": "accommodation",
"config": {}
},
{
"name": "student_life",
"config": {}
},
{
"name": "everyday_life",
"config": {}
},
{
"name": "other",
"config": {}
},
{
"name": "administrative",
"config": {}
},
{
"name": "culture",
"config": {}
},
{
"name": "tourism",
"config": {}
},
{
"name": "shared_comment",
"config": {}
},
{
"name": "specific_partnership",
"config": {}
}
"photos",
"insurance",
"transport",
"accommodation",
"student_life",
"everyday_life",
"other",
"administrative",
"culture",
"tourism",
"shared_comment",
"specific_partnership"
]
\ No newline at end of file
import json
import os
from base_app.models import User
from backend_app.models.tag import Tag
from base_app.models import User
from .loadGeneric import LoadGeneric
......@@ -17,12 +15,11 @@ class LoadTags(LoadGeneric):
self.admin = admin
def load(self):
tmp = os.path.join(os.path.realpath(__file__), "../../assets/tags.json")
tags_path = os.path.abspath(tmp)
with open(tags_path) as f:
tags = json.load(f)
for tag in tags:
t = Tag(name=tag["name"], config=tag["config"])
for tag_name in tags:
t = Tag(name=tag_name)
t.save()
self.add_info_and_save(t, self.admin)
......@@ -819,16 +819,7 @@ class Migration(migrations.Migration):
("has_pending_moderation", models.BooleanField(default=False)),
("name", models.CharField(max_length=200)),
("acronym", models.CharField(blank=True, default="", max_length=20)),
(
"logo",
models.URLField(
blank=True,
default="",
validators=[
backend_app.models.university.validate_extension_django
],
),
),
("logo", models.URLField(blank=True, default="")),
("website", models.URLField(blank=True, default="", max_length=300)),
("utc_id", models.IntegerField(unique=True)),
],
......
# Generated by Django 2.1.7 on 2019-04-17 19:25
import backend_app.validation.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("backend_app", "0001_squashed_0006_auto_20190416_2227")]
operations = [
migrations.RemoveField(model_name="tag", name="schema"),
migrations.AlterField(
model_name="tag",
name="name",
field=models.CharField(
max_length=100,
unique=True,
validators=[backend_app.validation.validators.TagNameValidator()],
),
),
]
# Generated by Django 2.1.7 on 2019-04-17 19:35
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [("backend_app", "0002_auto_20190417_2125")]
operations = [
migrations.RenameField(
model_name="campustaggeditem", old_name="custom_content", new_name="content"
),
migrations.RenameField(
model_name="citytaggeditem", old_name="custom_content", new_name="content"
),
migrations.RenameField(
model_name="countrytaggeditem",
old_name="custom_content",
new_name="content",
),
migrations.RenameField(
model_name="universitytaggeditem",
old_name="custom_content",
new_name="content",
),
]
# Generated by Django 2.1.7 on 2019-04-16 20:00
import backend_app.fields
import backend_app.validation.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("backend_app", "0004_auto_20190407_1010")]
operations = [
migrations.AlterField(
model_name="campus",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="campustaggeditem",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="citytaggeditem",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="countrydri",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="countryscholarship",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="countrytaggeditem",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="university",
name="logo",
field=models.URLField(
blank=True,
default="",
validators=[
backend_app.validation.validators.PathExtensionValidator(
["jpg", "jpeg", "png", "svg"]
)
],
),
),
migrations.AlterField(
model_name="universitydri",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="universityinfo",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="universityscholarship",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="universitysemestersdates",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
migrations.AlterField(
model_name="universitytaggeditem",
name="useful_links",
field=backend_app.fields.JSONField(
default=list,
validators=[backend_app.validation.validators.UsefulLinksValidator()],
),
),
]
# Generated by Django 2.1.7 on 2019-04-16 20:27
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [("backend_app", "0005_auto_20190416_2200")]
operations = [
migrations.RenameField(model_name="tag", old_name="config", new_name="schema")
]
......@@ -6,11 +6,12 @@ from backend_app.models.abstract.versionedEssentialModule import (
VersionedEssentialModuleSerializer,
VersionedEssentialModuleViewSet,
)
from backend_app.validators.tags import validate_content_against_config
from backend_app.validators.tags_config.useful_links import USEFULL_LINKS_CONFIG
from backend_app.validation.validators import UsefulLinksValidator
IMPORTANCE_LEVEL = (("-", "normal"), ("+", "important"), ("++", "IMPORTANT"))
useful_link_validator = UsefulLinksValidator()
class Module(VersionedEssentialModule):
"""
......@@ -24,7 +25,7 @@ class Module(VersionedEssentialModule):
title = models.CharField(default="", blank=True, max_length=150)
comment = models.CharField(default="", blank=True, max_length=5000)
useful_links = JSONField(default=list)
useful_links = JSONField(default=list, validators=[useful_link_validator])
importance_level = models.CharField(
max_length=2, choices=IMPORTANCE_LEVEL, default="-"
)
......@@ -38,18 +39,6 @@ class ModuleSerializer(VersionedEssentialModuleSerializer):
Custom serializer that performs checks on the Basic module filed
"""
def validate(self, attrs):
"""
Checks that the useful_links have been filled properly
"""
attrs = super().validate(attrs)
content = {"useful_links": attrs["useful_links"]}
config = {"useful_links": USEFULL_LINKS_CONFIG}
validate_content_against_config(config, content)
return attrs
class Meta:
model = Module
fields = VersionedEssentialModuleSerializer.Meta.fields + (
......
......@@ -3,7 +3,13 @@ from django.db import models
from backend_app.fields import JSONField
from backend_app.models.abstract.module import Module, ModuleSerializer, ModuleViewSet
from backend_app.models.tag import Tag
from backend_app.validators.tags import tagged_item_validation
from backend_app.validation.validators import TaggedItemValidator
VALIDATOR = TaggedItemValidator()
def validate_tagged_item(tag_name, content):
VALIDATOR(tag_name, content)
class TaggedItem(Module):
......@@ -12,7 +18,14 @@ class TaggedItem(Module):
"""
tag = models.ForeignKey(Tag, related_name="+", on_delete=models.PROTECT)
custom_content = JSONField(default=dict)
content = JSONField(default=dict)
def save(self, *args, **kwargs):
"""
Custom save function to ensure consistency of the content with the tag.
"""
validate_tagged_item(self.tag.name, self.content)
return super().save(*args, **kwargs)
class Meta:
abstract = True
......@@ -25,8 +38,7 @@ class TaggedItemSerializer(ModuleSerializer):
def validate(self, attrs):
attrs = super().validate(attrs)
tagged_item_validation(attrs)
validate_tagged_item(attrs["tag"].name, attrs["content"])
return attrs
......
from django.db import models
from backend_app.fields import JSONField
from backend_app.models.abstract.base import (
BaseModel,
BaseModelSerializer,
BaseModelViewSet,
)
from backend_app.permissions.app_permissions import ReadOnly
from backend_app.validation.utils import get_schema_for_tag
from backend_app.validation.validators import TagNameValidator
class Tag(BaseModel):
"""
Model to store available "tags" in the app.
Simple model to store available "tags" in the app.
The schema associated with each tag (name) must be defined in `tags_schemas_collection.json`
"""
name = models.CharField(max_length=100, unique=True)
config = JSONField(default=dict)
name = models.CharField(
max_length=100, unique=True, validators=[TagNameValidator()]
)
@property
def schema(self):
"""
Simple property to return the schema associated with a tag
:return:
"""
return get_schema_for_tag(self.name)
class TagSerializer(BaseModelSerializer):
class Meta:
model = Tag
fields = "__all__"
fields = BaseModelSerializer.Meta.fields + ("name", "schema")
class TagViewSet(BaseModelViewSet):
......
from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import models
from rest_framework.validators import ValidationError as RFValidationError
from backend_app.models.abstract.essentialModule import (
EssentialModule,
EssentialModuleSerializer,
EssentialModuleViewSet,
)
from backend_app.validators.tags import validate_extension
def validate_extension_django(url):
try:
validate_extension(settings.ALLOWED_PHOTOS_EXTENSION, url)
except RFValidationError:
raise ValidationError("Extension in URL doesn't correspond to an image")
from backend_app.validation.validators import PathExtensionValidator
class University(EssentialModule):
......@@ -26,7 +17,9 @@ class University(EssentialModule):
name = models.CharField(max_length=200)
acronym = models.CharField(max_length=20, default="", blank=True)
logo = models.URLField(
default="", blank=True, validators=[validate_extension_django]
default="",
blank=True,
validators=[PathExtensionValidator(settings.ALLOWED_PHOTOS_EXTENSION)],
)
website = models.URLField(default="", blank=True, max_length=300)
utc_id = models.IntegerField(unique=True)
......
......@@ -4,7 +4,6 @@ from django.db import models
from backend_app.models.abstract.module import Module, ModuleSerializer, ModuleViewSet
from backend_app.models.currency import Currency
from backend_app.models.university import University
from backend_app.permissions.app_permissions import ReadOnly
class UniversityInfo(Module):
......@@ -36,5 +35,5 @@ class UniversityInfoSerializer(ModuleSerializer):
class UniversityInfoViewSet(ModuleViewSet):
queryset = UniversityInfo.objects.all() # pylint: disable=E1101
serializer_class = UniversityInfoSerializer
permission_classes = (ReadOnly,)
# permission_classes = (ReadOnly,)
end_point_route = "universityInfo"
......@@ -6,7 +6,6 @@ from backend_app.models.abstract.taggedItem import (
TaggedItemViewSet,
)
from backend_app.models.university import University
from backend_app.permissions.app_permissions import IsOwner
class UniversityTaggedItem(TaggedItem):
......@@ -27,6 +26,6 @@ class UniversityTaggedItemSerializer(TaggedItemSerializer):
class UniversityTaggedItemViewSet(TaggedItemViewSet):
queryset = UniversityTaggedItem.objects.all() # pylint: disable=E1101
serializer_class = UniversityTaggedItemSerializer
permission_classes = (IsOwner,)
# permission_classes = (IsOwner,)
end_point_route = "universityTaggedItems"
filterset_fields = ("university",)
from django.test import TestCase
import pytest
from rest_framework.validators import ValidationError as RFValidationError
from django.core.validators import ValidationError as DJValidationError
from backend_app.validators.tags import validate_extension, validate_url
class ValidationUrlTestCase(TestCase):
@classmethod
def setUpTestData(cls):
cls.good = [
"https://www.epfl.ch/publ6d4772210.svg",
"http://google.fr/image.jpg",
]
cls.bad = ["google"]
pass
def test_validate_extension(self):
for url in self.good:
validate_extension(["svg", "jpg"], url)
for url in self.good + self.bad:
with pytest.raises(RFValidationError):
validate_extension(["png"], url)
def test_validate_url(self):
config_1 = {}
config_2 = {"validators": {"extension": ["jpg", "jpeg", "png", "svg"]}}
config_3 = {"validators": {"inconnu_au_bataillon": None}}
for url in self.good:
validate_url(config_1, url)
validate_url(config_2, url)
with pytest.raises(Exception):
validate_url(config_3, url)
for url in self.bad:
with pytest.raises(DJValidationError):
validate_url(config_1, url)
validate_url(config_2, url)
import pytest
from django.core.exceptions import ValidationError
from backend_app.validation.validators import (
UsefulLinksValidator,
PathExtensionValidator,
TagNameValidator,
PhotosValidator,
TaggedItemValidator,
)
class TestUsefulLinksValidator(object):
validator = UsefulLinksValidator()
def test_empty_array_ok(self):
data = []
self.validator(data)
def test_empty_object_not_ok(self):
data = {}
with pytest.raises(ValidationError):
self.validator(data)
def test_one_ok(self):
data = [{"description": "Coucou", "url": "https://utc.fr"}]
self.validator(data)
def test_fails_with_more_data(self):
data = [{"description": "Coucou", "url": "https://utc.fr", "lolilol": True}]
with pytest.raises(ValidationError):
self.validator(data)
def test_not_url(self):
data = [{"description": "Coucou", "url": "utc"}]
with pytest.raises(ValidationError):
self.validator(data)
class TestPhotosValidator(object):
validator = PhotosValidator()
def test_empty_array_ok(self):
data = []
self.validator(data)
def test_empty_object_not_ok(self):
data = {}
with pytest.raises(ValidationError):
self.validator(data)
def test_one_ok(self):
data = [{"title": "Coucou", "url": "https://utc.fr/logo.svg"}]
self.validator(data)
def test_not_ok(self):
data = [{"title": "Coucou", "url": "https://utc.fr/logosvg"}]
with pytest.raises(ValidationError):
self.validator(data)
def test_fails_with_more_data(self):
data = [{"title": "Coucou", "url": "https://utc.fr/logo.svg", "héhé": "houhou"}]
with pytest.raises(ValidationError):
self.validator(data)
class TestTaggedItemValidator(object):
validator = TaggedItemValidator()
def test_photos_tag_ok(self):
data = [{"title": "Coucou", "url": "https://utc.fr/logo.svg"}]
self.validator("photos", data)
def test_photos_tag_not_ok(self):
data = [{"title": "Coucou", "url": "//utc.fr/logo.svg"}]
with pytest.raises(ValidationError):
self.validator("photos", data)
def test_unknown_tag(self):
data = []
with pytest.raises(ValidationError):
self.validator("photossqsdknrncLQSKDJNCZIURYNCCH", data)
class TestPathExtensionValidator(object):
validator = PathExtensionValidator(["svg"])
def test_svg_ok(self):
self.validator("test.svg")
def test_no_extension(self):
with pytest.raises(ValidationError):
self.validator("test")
def test_multiple(self):
with pytest.raises(ValidationError):
self.validator("test.svg.zero")
class TestTagNameValidator(object):
validator = TagNameValidator()
def test_on_valid_tagname(self):
# Might need to change this at one point
# it transport is no more available
self.validator("transport")
def test_on_invalid_tagname(self):
# ie a tag name that doesn't have a matching schema
with pytest.raises(ValidationError):
self.validator("qmlskdjmlqfinzoieuncmlkqsdnkuy")
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "definitions.json",
"title": "REX-DRI sub-entity definition",
"description": "Schema to hold all the main definitions of JSON schemas from the REX-DRI project",
"definitions": {
"useful-links": {
"type": "array",
"items": {
"type": "object",
"properties": {
"url": {
"description": "The unique identifier for a product",
"type": "string",
"format": "uri",
"maxLength": 500
},
"description": {
"description": "Name of the product",
"type": "string",
"maxLength": 500