Commit 48a1e8ac authored by Florent Chehab's avatar Florent Chehab

setup(cleaned & debugging)

* **Added documentation regarding how to debug the backend and the frontend**
* Updated webpack config to ease debugging (no more mimify) and a bit better production config
* Added .env files to configure env variables in the project
* Updated backend image to support .env files
* Updated backend to use .env files
* Updated CI settings accordingly
* Added documentation about IDE setup
* Removed `.vscode` folder from repo

Fixes #106 #105 Linked to #66 for .env files
parent ede12d36
......@@ -7,28 +7,19 @@ stages:
- svg-gen-docu # required to be done before documentation and in separate stages
- documentation
variables:
ENV: DEV
SECRET_KEY: stupid_key_for_CI
DJANGO_ADMIN_USERNAME: admin
DJANGO_ADMIN_PASSWORD: admin
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
DB_HOST: postgres
DB_PORT: 5432
.only-default: &only-default
only:
- master
- merge_requests
check_back:
<<: *only-default
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend
services:
- postgres:10.5
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
before_script:
- make setup
script:
- cd backend && ./manage.py check
- cd ../documentation && make extract_django # Try to generate .dot files for the system architecture
......@@ -42,7 +33,7 @@ check_back:
check_front:
<<: *only-default
stage: check
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......@@ -53,12 +44,20 @@ check_front:
test_back:
<<: *only-default
stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
variables:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432 # We absolutely need this one since a gitlab runner will inject a similar variable; will cause tests to fail.
services:
- postgres:10.5
before_script:
- make setup
script:
- cd backend
- pytest base_app/ backend_app/ --cov-report html
- pytest base_app/ backend_app/ --cov --cov-config .coveragerc --cov-report html
artifacts:
paths:
- backend/htmlcov/
......@@ -69,7 +68,7 @@ test_back:
test_frontend:
<<: *only-default
stage: test
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......@@ -80,7 +79,7 @@ test_frontend:
flake8:
<<: *only-default
stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest
script:
- cd backend && flake8
tags:
......@@ -89,7 +88,7 @@ flake8:
eslint:
<<: *only-default
stage: lint
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend
image: registry.gitlab.utc.fr/rex-dri/rex-dri/frontend:latest
before_script:
- cd frontend && cp -R /usr/src/deps/node_modules .
script:
......
{
"files.exclude": {
"**/.git": true,
"**/__pycache__": true,
"**/*.pyc": true,
"backend/htmlcov": false,
"backend/.coverage": true,
"backend/.pytest_cache": true,
},
"cSpell.language": "en,fr-FR,fr",
"python.linting.flake8Enabled": true,
"python.formatting.provider": "black",
"python.linting.pylintEnabled": false,
"python.linting.enabled": true
}
.PHONY: documentation
up:
setup:
bash envs/init.sh
clear_setup:
rm envs/db.env
rm envs/django.env
up: setup
docker-compose up
docker-pull:
......@@ -13,7 +20,7 @@ reformat_backend:
docker-compose exec backend sh -c "cd backend && black ."
test_backend:
docker-compose exec backend sh -c "cd backend && pytest base_app/ backend_app/"
docker-compose exec backend sh -c "cd backend && pytest --cov --cov-config .coveragerc --cov-report term base_app/ backend_app/"
test_frontend:
docker-compose exec frontend sh -c "cd frontend && npm run test"
......
import os
from os.path import dirname, join
####################################
#
# Directory locations
#
####################################
BACKEND_ROOT_DIR = dirname(dirname(dirname(os.path.abspath(__file__))))
REPO_ROOT_DIR = dirname(BACKEND_ROOT_DIR)
ENVS_FILE_DIR = join(REPO_ROOT_DIR, "envs/")
......@@ -9,21 +9,15 @@ https://docs.djangoproject.com/en/2.0/ref/settings/
"""
import os
from os.path import dirname, normpath, join
from .app_settings import * # noqa: F403, F401 # We need to load app specific settings
import sys
from os.path import normpath, join
####################################
#
# Directory locations
#
####################################
from base_app.settings.dir_locations import BACKEND_ROOT_DIR, REPO_ROOT_DIR
from .app_settings import * # noqa: F403, F401 # We need to load app specific settings
BACKEND_ROOT_DIR = dirname(dirname(dirname(os.path.abspath(__file__))))
REPO_ROOT_DIR = dirname(BACKEND_ROOT_DIR)
# A stupid print to prevent pycharm from removing the import.
print("Moderation activated on backend ?", MODERATION_ACTIVATED) # noqa: F405
#
#
#####################################
#
# Django essential settings
......@@ -31,7 +25,11 @@ REPO_ROOT_DIR = dirname(BACKEND_ROOT_DIR)
#####################################
SECRET_KEY = os.environ["SECRET_KEY"]
try:
SECRET_KEY = os.environ["SECRET_KEY"]
except KeyError:
raise Exception("Env variable missing. Please run `make setup`")
ROOT_URLCONF = "base_app.urls"
......@@ -229,8 +227,8 @@ DATABASES = {
"NAME": os.environ["POSTGRES_DB"],
"USER": os.environ["POSTGRES_USER"],
"PASSWORD": os.environ["POSTGRES_PASSWORD"],
"HOST": os.environ["DB_HOST"],
"PORT": os.environ["DB_PORT"],
"HOST": os.environ["POSTGRES_HOST"],
"PORT": os.environ["POSTGRES_PORT"],
}
}
......
#!/usr/bin/env python3
import os
import sys
from os.path import join
from dotenv import load_dotenv
from base_app.settings.dir_locations import ENVS_FILE_DIR
if __name__ == "__main__":
# We make sure to use override = False to not override variables in the env.
load_dotenv(join(ENVS_FILE_DIR, "db.env"), override=False)
load_dotenv(join(ENVS_FILE_DIR, "django.env"), override=False)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base_app.settings.main")
try:
from django.core.management import execute_from_command_line
......
[pytest]
DJANGO_SETTINGS_MODULE = base_app.settings.main
python_files = tests.py test_*.py *_tests.py
addopts = --cov --cov-config .coveragerc --cov-report term
env_files =
../envs/db.env
../envs/django.env
......@@ -17,6 +17,7 @@ django-extensions==2.1.5
uwsgi==2.0.18
dotmap==1.3.4
django-webpack-loader==0.6.0
pytest-dotenv==0.4.0
# Direct dev depencies
ipython==7.3.0 # For a better Django shell
......
......@@ -13,7 +13,7 @@ services:
# Service for the backend app.
backend:
# Get the image from the registry
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.1.0
image: registry.gitlab.utc.fr/rex-dri/rex-dri/backend:v0.1.3
# To use a locally built one, comment above, uncomment bellow.
# build: ./backend
restart: on-failure
......@@ -24,20 +24,10 @@ services:
# Replicate the python server port
- 8000:8000
environment:
# Add environment variables
- ENV=DEV
- SECRET_KEY=please_change_me # Django secret key
# Django admin user param
- DJANGO_ADMIN_USERNAME=admin
- DJANGO_ADMIN_PASSWORD=admin
# DB parameters
- DB_HOST=database
- DB_PORT=5432
- POSTGRES_DB=postgres
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
# Specify host to wait for; ie we want to wait for the db for sure
- WAIT_HOSTS=database:5432
WAIT_HOSTS: database:5432 # For the 'wait' script, so that we are sure the db is up and running
env_file:
- ./envs/db.env
- ./envs/django.env
# Run the django developpement server on image startup.
command: /bin/sh -c "/wait && cd backend && ./entry.sh"
depends_on:
......@@ -49,11 +39,8 @@ services:
database:
# Use of classic image
image: postgres:10.5-alpine
environment:
# Set up the postgres ID
- POSTGRES_DB=postgres
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
env_file:
- ./envs/db.env
volumes:
# Add a volume to store the DB data.
- postgres_data:/var/lib/postgresql/data/
......
IDE setup
============
The project was initially developed with VScode but can obviously be enhanced with the IDE of your choice. At the time of this writing, the use of JetBrains IDEs is highly recommended as they are really powerful IDEs (**if you are a student you can get their *ultimate* versions for free**).
In this short documentation, some configurations "issues" regarding your choice of IDE is addressed. Also, don't miss the separate documentation about the configuration of your [IDE for debugging](Technologies/debugging.md).
?> :information_desk_person: JetBrains IDEs are cranked with features and plugin which can make them a bit slow on some system. More than 75% of those plugin are completely useless for the project so you should dive into the settings and deactivate them.
!> In JetBrain IDEs you should also deactivate *auto-save* since this would cause useless recompilation of the frontend (or useless restart of the backend). Check [this link](https://intellij-support.jetbrains.com/hc/en-us/community/posts/207054215-Disabling-autosave).
## For the backend
For the backend the use of [`PyCharm`](https://www.jetbrains.com/pycharm/?fromMenu) is recommended.
In this case you only need to configure the correct python interpreter:
* Open the settings (`CTRL + Alt + S`) go to `project: backend` and `Project interpreter`.
* Select `Docker` and put `registry.gitlab.utc.fr/rex-dri/rex-dri/backend:latest` for the image name.
This will give PyCharm access to the project python's dependencies.
Further configuration is required to run the project (optionally in debug mode) from the IDE. This is documented [here](Technologies/debugging.md).
## For the frontend
For the frontend the use of [`WebStorm`](https://www.jetbrains.com/webstorm/?fromMenu) is recommended.
No special configuration is required, except you might:
* Need to install `Node.js` on your system
* You should install the Node.js dependencies locally (they are in the frontend docker image, but they can't be used through the IDE this time). So run `npm ci`.
## For general purposes
`VS Code` is still a really nice general purposes IDE.
Debugging
==========
:hospital: Being able to debug easily a program is essential for a smooth developer experience. :hospital:
Using `console.log(...)` or `print(...)` should be a thing of the past as it makes us inefficient. Hopefully the `rex-dri` project is *debugger*-friendly and in this section we will see how to set-up those awesome tools! :tada:
## Backend
### Using PyCharm
#### Simple run
Before diving into the configuration for debugging we need to configure the project to be able to run it directly from the IDE.
* In the menu, click `run` -> `add configuration`
* Select `Django server`, and give the configuration a thoughtful name :smile:.
* Change the host to `0.0.0.0` and the port to `18000` (a port that should be free on the container).
* (The python interpreter should be a *remote* docker one, configured in the project settings, see [here](GettingStarted/IDE_setup.md))
* Then you need to edit the `Docker container settings`:
* Set the `network mode` to `rex-dri_default` (the network that should be created by docker-compose, this is so that we can access be on the same virtual network of the databse)
* Add one entry in `Port bindings`:
* *Container port*: `18000`
* *Protocol*: `tcp`
* *Host IP*: *(nothing)*
* *Host Port*: `18081` *(or the one you prefer and that is free)*
* Finally set the volume binding as follow:
* *Container path*: `/usr/src/app`
* *Host path*: `/home/xxxxx/git/rex-dri/` (or wherever is the **root** of the project: the whole repository is needed for the backend to work)
At the end the settings line should look like something like this: `--net="rex-dri_default" -p :18081:18000/tcp -v /home/xxxxx/git/rex-dri/:/usr/src/app`
:tada:
?> :information_desk_person: So if you run this, the website will be accessible on url: [http://localhost:18081/](http://localhost:18081/). Make sure to have made a `make up` before (or at least a `docker-compose up database frontend` to have only the database and frontend services up and running). With `make up` you will have 2 backends running (one on port `8000` and one on port `18081`); but that's fine.
?> :information_desk_person: If for some reasons you have errors when starting the server from PyCharm, try to remove all your remote interpreters and add the docker-based interpreter again.
#### Debugging the app
Once you have followed the previous setup, you can just put breakpoints in the code and lunch a debugging session. :tada:
#### Debugging the tests
The backend tests uses `pytest` so first we need to deactivate the default Django test runner:
* Open the settings -> `Languages & Frameworks` -> `Django`
* Check `Do not use Django test runner`
* Also in `Tools` -> `Python intgrated tools` set the *default test runner* to `pytest`.
Make sure you install the `EnvFile` plugin (from the IDE settings).
Then:
* In the menu, click `run` -> `add configuration`
* Edit the **default** pytest template in `templates` -> `Python tests` -> `pytest`.
* In the `EnvFile` tab add two entries for `db.env` and `django.env` that are in the `envs` repo directory.
* In the `Configuration` tab make sure the correct remote interpreter is selected, then set the `Docker container settings` as follow:
* Set the `network mode` to `rex-dri_default` (the network that should be created by docker-compose, this is so that we can access be on the same virtual network of the databse)
* Finally set the volume binding as follow:
* *Container path*: `/usr/src/app`
* *Host path*: `/home/xxxxx/git/rex-dri/` (or wherever is the **root** of the project: the whole repository is needed for the backend to work)
At the end the settings line should look like something like this: `--net="rex-dri_default" -v /home/xxxxx/git/rex-dri/:/usr/src/app`.
You can now add breakpoints and run the tests (or a specific one) in debug mode. :tada:
## Frontend app
Using a debugger for the frontend is extremely easy.
### Using (only) the debugger of your browser
Steps:
1. Hard-code a breakpoint (or multiple breakpoints) wherever you want with the `debugger;` keyword:
```js
const a = 2;
debugger;
```
*NB: the file should be automatically rebuilt if you have used the `make up` command.*
2. Make sure to have your developer console opened `F12` in your browser.
3. As soon as the code with the breakpoint is run, the execution will stopped and you should be brought in the debugger with a lot of information such as the variables currently defined, the *callstack*, etc.
?> :information_desk_person: If you want to debbug what is going on in a `React` component, you should use the [`React Developer Tools`](https://chrome.google.com/webstore/detail/react-developer-tools/fmkadmapgofadopljbjfkapdkoienihi) Chrome/Chromium extension. This extension will add another panel in your browser's developer tools to easily navigate React's virtual DOM and your components `props`/`state`.
### Using your IDE
You may want to use your IDE debugging console too. Here are the step to follow depending on your IDE.
#### WebStorm
1. (In the top menu) click `Run` -> `Edit Configurations...` -> `+` (add) -> `JavaScript Debug`
2. Give it meaningful name and in the URL put: `http://localhost:8000/` (remember that our frontend is served through the backend)
3. Click `Apply` and `Ok`.
4. Add a breakpoint somewhere in the code (click next to the line number; you should see a dot :red_circle: next to it when it is added)
5. Then click on the green :bug: next to the play button in the top row (make sure the configuration you just defined is the one selected)
6. Your browser will open and once the breakpoint is hit, you will have access to all the debugging information directly in WebStorm. :tada:
*Don't forget to stop the debug session once you are done.*
?> :information_desk_person: If your are using Chromium on linux, in the path of the Chrome browser config, you should be fine with `chromium-browser` value.
-----
You may want to debug some tests at some point. Add breakpoints where you want and simply run the test in debug mode: you whould have a green play button on the left of the first line of your test. Click on it and click `Debug ...`. That's all.
#### Visual Studio Code
0. Install the [`Debugger For Chrome` extension](https://marketplace.visualstudio.com/items?itemName=msjsdiag.debugger-for-chrome)
1. In the top menu, click `Debug` -> `Add configuration...` and select the `Chrome` environment.
2. Make sure the configuration for `url` points to `http://localhost:8000`. Bellow is an example configuration to be used with `chromium` (tested on Fedora 29).
```json
{
"version": "0.2.0",
"configurations": [
{
"type": "chrome",
"request": "launch",
"name": "Launch Chrome against localhost",
"url": "http://localhost:8000",
"webRoot": "${workspaceFolder}",
"runtimeExecutable": "/usr/bin/chromium-browser" // make sure this leads to your chrome installation
}
]
}
```
3. Add breakpoints in your code
4. And lunch it. In your browser, once the code with the breakpoint is executed, the execution will stop and you will be able to debug in VSCode. :tada:
......@@ -5,6 +5,7 @@
* [Introduction](GettingStarted/introduction.md)
* [Set-up](GettingStarted/set-up.md)
* [Initializing data](GettingStarted/init_data.md)
* [IDE Setup](GettingStarted/IDE_setup.md)
- Application documentation
......@@ -31,6 +32,7 @@
- Comments about technologies used
* [Use of `Docker`](Technologies/docker.md)
* [Debugger](Technologies/debugging.md)
- Other
......
......@@ -37,6 +37,7 @@
<script src="//unpkg.com/prismjs/components/prism-python.min.js"></script>
<script src="//unpkg.com/prismjs/components/prism-javascript.min.js"></script>
<script src="//unpkg.com/prismjs/components/prism-jsx.min.js"></script>
<script src="//unpkg.com/prismjs/components/prism-json.min.js"></script>
<script src="//unpkg.com/prismjs/components/prism-sql.min.js"></script>
</body>
......
db.env
django.env
POSTGRES_DB=postgres
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
# name of the service in docker-compsoe
POSTGRES_HOST=database
# default postgres port
POSTGRES_PORT=5432
# Set the env state
ENV=DEV
# Django secret key
SECRET_KEY=please_change_me
# default django admin username
DJANGO_ADMIN_USERNAME=admin
# and password
DJANGO_ADMIN_PASSWORD=admin
#!/bin/bash
if [ ! -f `dirname $0`/db.env ]; then
echo "db.env file doesn't exist, creating it with example/dev settings"
cp `dirname $0`/db.ex.env `dirname $0`/db.env
else
echo "db.env file already exists, not recreating it."
fi
if [ ! -f `dirname $0`/django.env ]; then
echo "db.env file doesn't exist, creating it with example/dev settings"
cp `dirname $0`/django.ex.env `dirname $0`/django.env
else
echo "django.env file already exists, not recreating it."
fi
import { createStore, applyMiddleware } from "redux";
import {applyMiddleware, createStore} from "redux";
import rootReducer from "./reducers/index";
import thunkMiddleware from "redux-thunk";
import { createLogger } from "redux-logger";
import {createLogger} from "redux-logger";
import {
SAVE_APP_COLOR_PICKER,
SAVE_APP_THEME
} from "./actions/action-types";
import {SAVE_APP_COLOR_PICKER, SAVE_APP_THEME} from "./actions/action-types";
const loggerMiddleware = createLogger(
{
collapsed:
(getState, action, logEntry) => { // eslint-disable-line no-unused-vars
const { type } = action;
const {type} = action;
if (type == SAVE_APP_COLOR_PICKER || type == SAVE_APP_THEME) {
return false;
}
......@@ -21,11 +18,20 @@ const loggerMiddleware = createLogger(
}
});
let middlewares = [
thunkMiddleware, // lets us dispatch() functions
];
// A bit of optimization for production builds
// eslint-disable-next-line no-undef
if (process.env.NODE_ENV !== "production") {
middlewares.push(loggerMiddleware); // neat middleware that logs actions
}
const store = createStore(
rootReducer,
applyMiddleware(
thunkMiddleware, // lets us dispatch() functions
loggerMiddleware // neat middleware that logs actions
...middlewares
)
);
......
......@@ -2,6 +2,9 @@ const merge = require("webpack-merge");
const webpackConfig = require("./webpack.config.base");
module.exports = merge(webpackConfig, {
devtool: "source-map",
mode: "production",
optimization: {
minimize: true,
nodeEnv: "production",
}
});
......@@ -3,17 +3,19 @@ const webpackConfig = require("./webpack.config.base");
const webpack = require("webpack");
const devConfig = merge(webpackConfig, {
devtool: "eval",
mode: "development",
devtool: "eval-source-map",
entry: {
main: [
"webpack-dev-server/client?http://localhost:3000",
"webpack/hot/only-dev-server",
]
},
optimization: {
minimize: false,
},
output: {
publicPath: "http://localhost:3000/dist/bundles/",
path: __dirname + "/dist/bundles",
},
plugins: [
new webpack.HotModuleReplacementPlugin(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment