Update architecture, added config file and generating

.gitignore

@@ -2,3 +2,4 @@
 .ipynb_checkpoints
 proj_env
 data
+__pycache__
\ No newline at end of file

.vscode/settings.json

+{
+    "python.pythonPath": "${workspaceFolder}/proj_env/bin/python",
+    "python.linting.pylintEnabled": true,
+    "python.linting.flake8Enabled": true
+}
\ No newline at end of file

analysis/analysis.ipynb → analysis.ipynb

 %% Cell type:raw id: tags:
 
 <script>
   function code_toggle() {
     if (code_shown){
       $('div.input').hide('200');
       $('#toggleButton').val('Afficher le code source')
     } else {
       $('div.input').show('200');
       $('#toggleButton').val('Cacher le code source')
     }
     code_shown = !code_shown
   }
 
   $( document ).ready(function(){
     code_shown=false;
     $('div.input').hide()
   });
 </script>
 <form action="javascript:code_toggle()">
   <input type="submit" id="toggleButton" value="Afficher le code source" style="margin-left: auto;margin-right: auto; display:block;" class="btn btn-info">
 </form>
 
 %% Cell type:markdown id: tags:
 
 <center>
 <img src="https://gitlab.utc.fr/LaTeX-UTC/Graphismes-UTC/raw/56dd9762de926727aa45c8279dcdf54a753335c8/logos/UTC/logo_UTC.png" alt="Logo UTC" width="200px"/>
 <br>
-<H1>Observatoire des évaluations d'UVs</H1>
+<font size="15">Observatoire des évaluations d'UVs</font>
 </center>
 
 %% Cell type:code id: tags:
 
 ``` python
-import socket
-
-def is_connected():
-    hostname = "www.utc.fr"
-    try:
-        host = socket.gethostbyname(hostname)
-        s = socket.create_connection((host, 80), 2)
-        return True
-    except:
-        pass
-    return False
+import yaml
+import warnings
+import os.path
+from src.generate_data import generate_data_file
+
+CONFIGURATION = yaml.load(open('./config.yml'))
+ALL_SEMESTERS = {info['sem']:info['end_date'] for info in CONFIGURATION["other_semesters"]}
+ALL_SEMESTERS[CONFIGURATION["main_semester"]['sem']] = CONFIGURATION["main_semester"]['end_date']
+
+for semester in ALL_SEMESTERS.keys():
+    if not os.path.isfile('./data/' + semester + '.json'):
+        warnings.warn("Il manque des fichiers de données des évaluations d'UVs pour le semestre "
+                      + semester
+                      + ". Nous allons en générer pour pouvoir continer.")
+        generate_data_file(semester)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # For auto completion inside jupyter notebook
 %config IPCompleter.greedy=True
 
+from src.tools import is_connected # simple function to check if we have an internet connectection
 import json
 from collections import OrderedDict
 from datetime import datetime
 
 from IPython.core.display import HTML
 from IPython.core.display import Markdown
 
 # For plotting data
 from plotly.offline import init_notebook_mode, plot, iplot
 import plotly.graph_objs as go
 
 if is_connected():
     # do not include plotly js directly
     init_notebook_mode(connected=True)
 else:
     # include plotly js directly
     import cufflinks as cf
     cf.set_config_file(offline=True, world_readable=True, theme='ggplot')
 
 # pandas and numpy for managing data
 import pandas as pd
 import numpy as np
 
 
 # import data
-data_json = json.load(open('../data/A2017.json'))
+data_json = json.load(open('./data/A2017.json'))
 DATE = data_json["date"]
 semester = data_json["semester"]
 data_dict = OrderedDict(data_json["data"])
 
 # first we convert the date and time
 for uv, description in data_dict.items():
     if description["date_review"] is not None:
         description["date_review"] = datetime.strptime(description["date_review"], '%d/%m/%Y')
 
 # and we make it pandas dataframe
 data_pd = pd.DataFrame.from_dict(data_dict, orient = "index")
 ```
 
 %% Cell type:markdown id: tags:
 
-## Généralités
+# Généralités
 
-### Présentation des données
+## Présentation des données
 
 %% Cell type:code id: tags:
 
 ``` python
 Markdown("Les données datent du : " + str(DATE))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(list(data_pd))
 ```
 
 %% Cell type:markdown id: tags:
 
 Voici un exemple de ligne (il y a une ligne par UV) :
 <div class="alert alert-success">
   <strong>Success!</strong> Indicates a successful or positive action.
 </div>
 
 %% Cell type:code id: tags:
 
 ``` python
 print(data_pd.iloc[0])
 ```
 
 %% Cell type:markdown id: tags:
 
 ### Taux de réponse
 
 %% Cell type:code id: tags:
 
 ``` python
 nb_uvs = len(data_pd.index)
 print("Nombre d'UVs étudiées : "+str(nb_uvs))
 
 uvs_with_evals = data_pd.loc[pd.notna(data_pd["date_review"])]
 nb_uvs_with_evals = len(uvs_with_evals.index)
 print("Nombre d'UVs où le responsable a visualisé les évaluations : "+str(nb_uvs_with_evals))
 print("Taux de visualisation : "+"{0:.0f}%".format(float(nb_uvs_with_evals)/nb_uvs * 100))
 ```
 
 %% Cell type:markdown id: tags:
 
 ### Délai de réponse des responsables d'UVs
 
 %% Cell type:code id: tags:
 
 ``` python
 graph_data = [
     go.Scatter(
         x=uvs_with_evals.sort_values(by=['date_review'])['date_review'],
         y=[100*float(i+1)/nb_uvs for i,e in enumerate(uvs_with_evals.index)],
         text=uvs_with_evals.sort_values(by=['date_review']).index
         )
 ]
 layout = go.Layout(
     title = "Délai de réponse",
     yaxis=dict(
         range=[0, 100],
         title='Taux de réponse',
     ),
     xaxis=dict(
         title="Date"
     )
 
 )
 
 iplot(go.Figure(data=graph_data, layout=layout))
 ```
 
 %% Cell type:markdown id: tags:
 
 # Analyse quantitative
 
 %% Cell type:code id: tags:
 
 ``` python
 def get_stats_uv_question(stats, question):
     q = str(question)
     return stats[q]["++"], stats[q]["+"], stats[q]["-"], stats[q]["--"]
 
 
 nb_pp, nb_p, nb_m, nb_mm = 0, 0, 0, 0
 for i, row in data_pd.iterrows():
     for q in range(1,11):
         a, b, c, d = get_stats_uv_question(row["stats"],q)
         nb_pp += a
         nb_p += b
         nb_m += c
         nb_mm += d
 
 print(nb_pp, nb_p, nb_m, nb_mm)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 color_mm = 'rgb(255,48,48)'
 color_m = 'rgb(255,165,0)'
 color_p = 'rgb(144,238,144)'
 color_pp = 'rgb(48,221,48)'
 
 q_lists_pd = [str(i) for i in range(1,11)] # TODO BETTER
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 def get_stat_by_question(question, s = "++"):
     q = str(question)
     res = 0
     for i, row in data_pd.iterrows():
         res += row["stats"][q][s]
     return res
 
 
 
 q_list = ["q"+str(i) for i in range(1,11)]
 
 
 
 trace1 = go.Bar(
     x=q_list,
     y=[get_stat_by_question(i, "--") for i in range(1,11)],
     name='--',
     marker=dict(color=color_mm)
 )
 trace2 = go.Bar(
     x=q_list,
     y=[get_stat_by_question(i, "-") for i in range(1,11)],
     name='-',
     marker=dict(color=color_m)
 )
 trace3 = go.Bar(
     x=q_list,
     y=[get_stat_by_question(i, "+") for i in range(1,11)],
     name='+',
     marker=dict(color=color_p)
 )
 trace4 = go.Bar(
     x=q_list,
     y=[get_stat_by_question(i, "++") for i in range(1,11)],
     name='++',
     marker=dict(color=color_pp)
 )
 
 data = [trace1, trace2, trace3, trace4]
 layout = go.Layout(
     barmode='stack',
     title="Total cumulé de l'ensemble des évaluations d'UVs pour chaque question"
 )
 
 
 fig = go.Figure(data=data, layout=layout)
 iplot(fig)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 def get_mm_per_row(row, relative = True):
     nb_mm = 0
     # for q in q_lists_pd :
     #    nb_mm += row["stats"][q]["--"]
 
     nb_mm = row["stats"]["10"]["--"]
 
     if relative:
         nb_eval = sum([row["stats"]["1"][i] for i in ["--","-","+","++"] ])
         # return nb_mm / float(nb_eval*len(q_list))
         return nb_mm / float(nb_eval)
     else :
         return nb_mm
 
 tmp = data_pd.apply(get_mm_per_row, axis=1)
 
 graph_data = [
     go.Scatter(
         x=tmp.sort_values().index,
         y=tmp.sort_values(),
         )
 ]
 layout = go.Layout(
     title="Pourcentage d'avis très négatif (--) sur l'appréciation finale de l'UV (question 10 des évaluations)"
 )
 
 
 fig = go.Figure(data=graph_data, layout=layout)
 iplot(fig)
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Remarques
 
 Pour la suite de l'analyse il est plus simple d'affecter une note numérique à chaque évaluation. Pour se faire il a été décidé d'utiliser la table de conversion _symétrique_ suivante :
 
 | Sigle | Valeur associée |
 |-------|-----------------|
 | ++    | +3              |
 | +     | +1              |
 | -     | -1              |
 | --    | -3              |
 
 %% Cell type:code id: tags:
 
 ``` python
 list_answers = ["--","-","+","++"]
 list_coeffs = [-3,-1,1,3]
 
 def get_nb_eval(row):
     return sum([row["stats"]["1"][i] for i in list_answers])
 
 def get_mark_q(row, q):
     if type(q) is int:
         q = str(q)
     return sum([row["stats"][q][list_answers[i]] * list_coeffs[i] for i in range(len(list_coeffs))]) / get_nb_eval(row)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 tmp = data_pd.apply((lambda r  : get_mark_q(r,10)), axis=1)
 
 graph_data = [
     go.Scatter(
         x=tmp.sort_values(ascending = False).index,
         y=tmp.sort_values(ascending = False),
         )
 ]
 
 layout = go.Layout(
     title="Visualisation de l'appréciation globale de chaque UV (question 10) sous forme de note moyenne"
 )
 
 fig = go.Figure(data=graph_data, layout=layout)
 iplot(fig)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # Tweak css for displaying data
 
 from IPython.core.display import HTML
-style = open('./style.css').read()
+style = open('./src/style.css').read()
 HTML("<style>"+style+"</style>")
 ```
+
+%% Cell type:code id: tags:
+
+``` python
+```

config.yml

+main_semester: 
+  sem: A2017
+  end_date: 13/01/2018
+
+other_semesters: 
+  - sem: P2017
+    end_date: 30/06/2017
+    
\ No newline at end of file

src/generate_data.py

+import datetime
+from datetime import date
+import string
+import random
+import json
+import os.path
+
+
+def generate_data_file(semester):
+    """
+    Function for generating a random data file to 
+    be used in analysis
+    """
+    TODAY = datetime.datetime.today().strftime('%d/%m/%Y')
+    output = {
+        'date': TODAY,
+        'semester': semester,
+        'data': {}
+    }
+
+    def random_eval(nb_evals):
+        stats = {}
+        for i in range(1, 10):
+            pp = random.randint(0, nb_evals)
+            p = random.randint(0, nb_evals - pp)
+            m = random.randint(0, nb_evals - pp - p)
+            mm = nb_evals - pp - p - m
+            stats[str(i)] = {
+                '++': pp,
+                '+': p,
+                '-': m,
+                '--': mm
+            }
+        return stats
+
+    def random_date():
+        start_date = date.today().toordinal() - 60
+        end_date = date.today().toordinal()
+        random_day = date.fromordinal(random.randint(start_date, end_date))
+        return random_day.strftime('%d/%m/%Y')
+
+    NB_UVS = 200
+    for uv in range(1, NB_UVS):
+        uv_code = 'UV' + str(uv)
+        nb_etu_registered = random.randint(10, 100)
+        nb_etu_abs = random.randint(1, 9)
+        nb_etu_passed = random.randint(1, nb_etu_registered - nb_etu_abs)
+        nb_evals = random.randint(1, nb_etu_registered - nb_etu_abs)
+        date_review = random.choice([None, random_date()])
+        comment = None
+        if date_review:
+            comment = random.randint(0, 500)
+
+        output['data'][uv_code] = {
+            'name': ''.join(random.choices(string.ascii_uppercase, k=40)),
+            'nb_etu_registered': nb_etu_registered,
+            'nb_etu_abs': nb_etu_abs,
+            'nb_etu_passed': nb_etu_passed,
+            'nb_evals': nb_evals,
+            'stats': random_eval(nb_evals),
+            'date_review': date_review,
+            'comment': comment
+        }
+
+    fp = os.path.realpath(__file__) + '/../../data/' + semester + '.json'
+    with open(os.path.abspath(fp), 'w') as outfile:
+        json.dump(output, outfile)

src/tools.py

+import socket
+
+
+def is_connected():
+    hostname = "www.utc.fr"
+    try:
+        host = socket.gethostbyname(hostname)
+        socket.create_connection((host, 80), 2)
+    except:  # noqa: E722
+        pass
+    return False