Skip to content

GitLab

Commit 247ec043 authored by Romain Creuzenet's avatar Romain Creuzenet
Browse files

synchronisation & clean server

parent cb5c576e
Hide whitespace changes
Inline Side-by-side
Showing with 88 additions and 56 deletions
.gitignore
View file @ 247ec043
/__pycache__/*
/env_nf26/*
*/.*
/out
/out/*
.idea/workspace.xml
View file @ 247ec043
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="dec891dc-2fad-4291-af33-64d4fd64029d" name="Default Changelist" comment="" />
<list default="true" id="dec891dc-2fad-4291-af33-64d4fd64029d" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parameters.py" beforeDir="false" afterPath="$PROJECT_DIR$/parameters.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -22,8 +27,8 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="393">
<caret line="155" column="24" selection-start-line="155" selection-start-column="24" selection-end-line="155" selection-end-column="24" />
<state relative-caret-position="589">
<caret line="205" column="36" selection-start-line="205" selection-start-column="36" selection-end-line="205" selection-end-column="36" />
<folding>
<element signature="e#45#75#0" expanded="true" />
</folding>
......@@ -43,8 +48,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/parameters.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-82">
<caret line="26" column="19" selection-start-line="26" selection-start-column="19" selection-end-line="26" selection-end-column="19" />
<state relative-caret-position="513">
<caret line="27" selection-start-line="27" selection-end-line="27" />
<folding>
<element signature="e#62#99#0" expanded="true" />
</folding>
......@@ -204,6 +209,13 @@
<entry file="file://$PROJECT_DIR$/data/LFOI_2001_2010.csv">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/download_data.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-804">
<caret line="5" column="52" selection-start-line="5" selection-start-column="52" selection-end-line="5" selection-end-column="52" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/create_table.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="811">
......@@ -213,25 +225,18 @@
</entry>
<entry file="file://$PROJECT_DIR$/parameters.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-82">
<caret line="26" column="19" selection-start-line="26" selection-start-column="19" selection-end-line="26" selection-end-column="19" />
<state relative-caret-position="513">
<caret line="27" selection-start-line="27" selection-end-line="27" />
<folding>
<element signature="e#62#99#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/download_data.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-804">
<caret line="5" column="52" selection-start-line="5" selection-start-column="52" selection-end-line="5" selection-end-column="52" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="393">
<caret line="155" column="24" selection-start-line="155" selection-start-column="24" selection-end-line="155" selection-end-column="24" />
<state relative-caret-position="589">
<caret line="205" column="36" selection-start-line="205" selection-start-column="36" selection-end-line="205" selection-end-column="36" />
<folding>
<element signature="e#45#75#0" expanded="true" />
</folding>
......
main.py
View file @ 247ec043
"""File to execute to show results"""
# Data
from parameters import SESSION
from parameters import SESSION, DIR_OUT, START, END
# Basic
import re
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import warnings
import re
import os
# Stats
import statsmodels.graphics as stm_graphs
import pandas as pd
......@@ -36,6 +37,7 @@ def ask_q(possibilities, text=">>> "):
def ask_d(text=">>> "):
"""Demande une date"""
print("Entrez une date sous la forme YYYY-MM-DD HH:mm")
print("Comprise entre {} et {}".format(START.strftime('%Y-%m-%d'), END.strftime('%Y-%m-%d')))
date_parser = re.compile(r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2}) (?P<hour>\d{2}):(?P<minute>\d{2})")
match = None
......@@ -49,7 +51,7 @@ def ask_d(text=">>> "):
return result
def choose_attr():
def chose_attr():
"""Permet de demander un attribut dans la table"""
# Search element
decision = {
......@@ -73,7 +75,11 @@ class Manager:
table = None # table name use by the function
def run(self):
"""Chose"""
"""Chose objective"""
# Initialisation
os.makedirs(DIR_OUT, exist_ok=True)
# Chose objective
print("Choisissez ce que vous voulez faire")
print("\t1 - Pour un point donné de l’espace, je veux pouvoir avoir un historique du passé")
print("\t2 - À un instant donné je veux pouvoir obtenir une carte me représentant n’importe quel indicateur")
......@@ -89,6 +95,7 @@ class Manager:
def historic(self):
self.table = "TABLE_SPACE"
print("=== Choix 1 : Historique ===")
# Search station
stations = []
......@@ -101,10 +108,10 @@ class Manager:
print()
station = ask_q(stations)
attr = choose_attr()
attr = chose_attr()
ts = pd.Series()
query = "SELECT time ,{} FROM {} WHERE station={}".format(attr, self.table, station.__repr__())
query = "SELECT time, {} FROM {} WHERE station={}".format(attr, self.table, station.__repr__())
for row in execute_query(query):
value = getattr(row, attr)
if value is None:
......@@ -117,25 +124,29 @@ class Manager:
plt.xticks(rotation=90)
plt.plot(ts, label=attr)
plt.title("Donnees de " + str(attr) + " pour la station : " + station)
plt.title("Donnees de {} pour la station : {}".format(attr, station))
plt.legend()
plt.show()
plt.savefig('./out/graph_' + station + '_' + str(attr) + '.png')
path = os.path.join(DIR_OUT, 'graph_{}_{}.png'.format(station, attr))
plt.savefig(path)
res = stm.tsa.seasonal_decompose(ts, freq=15, extrapolate_trend='freq')
res.plot()
plt.show()
plt.savefig('./out/decompose_' + station + '_' + str(attr) + '.png')
path = os.path.join(DIR_OUT, 'decompose_{}_{}.png'.format(station, attr))
plt.savefig(path)
stm_graphs.tsaplots.plot_acf(ts, lags=30)
plt.show()
plt.savefig('./out/acf_' + station + '_' + str(attr) + '.png')
path = os.path.join(DIR_OUT, 'acf_{}_{}.png'.format(station, attr))
plt.savefig(path)
def map(self):
self.table = "TABLE_TIME"
print("=== Choix 2 : Map ===")
date = ask_d()
attr = choose_attr()
attr = chose_attr()
plt.figure(figsize=(14, 14))
the_map = Basemap(projection='mill', llcrnrlat=25.281898, llcrnrlon=-18.42, urcrnrlat=48.08, urcrnrlon=10.35,
......@@ -150,41 +161,54 @@ class Manager:
the_map.drawmeridians(np.arange(0, 360, 30))
the_map.drawparallels(np.arange(-90, 90, 30))
try:
query = "SELECT station, lon, lat, {} FROM {} WHERE time={}".format(attr, self.table, date)
for row in execute_query(query):
if getattr(row, "station") is None:
continue
x, y = the_map(row.lon, row.lat)
value = getattr(row, attr)
plt.plot(x, y, 'go')
plt.annotate(round(value, 1), (x, y))
plt.title('Map')
plt.show()
plt.savefig('./out/map.png')
except:
dateb = list(date)
dateb[3] = 0
dateb[4] = 0
dateb = tuple(dateb)
datee = list(date)
datee[3] = 23
datee[4] = 59
datee = tuple(dateb)
print("Seul ces heures sont disponibles pour ce jour")
query = "SELECT DISTINCT time FROM {} WHERE time>={} AND time <= {} ALLOW FILTERING".format(self.table,
dateb, datee)
date_ok = False # The date is valide
query = "SELECT station, lon, lat, {} FROM {} WHERE time={}".format(attr, self.table, date)
for row in execute_query(query):
if getattr(row, "station") is None or getattr(row, attr) is None:
continue
date_ok = True
x, y = the_map(getattr(row, "lon"), getattr(row, "lat"))
value = getattr(row, attr)
plt.plot(x, y, 'go')
plt.annotate(round(value, 1), (x, y))
title = "Map {} du {}".format(attr, datetime(*list(date)).strftime('%Y-%m-%d %H:%M'))
plt.title(title)
plt.show()
for elt in ' :-':
title = title.replace(elt, '_')
path = os.path.join(DIR_OUT, title.lower() + '.png')
plt.savefig(path)
# If date is wrong, show some dates for this day
if not date_ok:
date_begin = list(date)
date_begin[3] = date_begin[4] = 0 # set hours and minutes at 0
date_begin = tuple(date_begin)
date_end = list(date)
date_end[3] = 23
date_end[4] = 59
date_end = tuple(date_end)
print("Seules ces heures sont disponibles pour ce jour")
query = "SELECT DISTINCT time FROM {} WHERE time >= {} AND time <= {} ALLOW FILTERING".format(
self.table,
date_begin,
date_end,
)
for row in execute_query(query):
print(getattr(row, "time"))
resp = list(getattr(row, "time"))
print(str(resp[3]).zfill(2) + ":" + str(resp[4]).zfill(2), end=" - ")
def cluster(self):
self.table = "TABLE_TIME"
print("=== Choix 3 : CLUSTER ===")
# Ask Date
print("Entrez la date de départ")
print("La date de départ :")
date_b = ask_d()
print("Entrez la date de fin")
print("Entrez la date de fin sous la forme YYYY-MM-DD HH:mm")
date_e = ask_d()
print("Entrez le nombre de cluster voulus")
nb_clust = ask_cluster()
......
parameters.py
View file @ 247ec043
......@@ -20,10 +20,11 @@ for key in KEY_SPACE:
else:
break
if SESSION is None:
raise NoHostAvailable("Erreur de connextion à cassandra")
raise NoHostAvailable("Erreur de connection à cassandra")
BASE_DIR = os.path.dirname(os.path.abspath(__file__)) # path of project
DIR_DATA = os.path.join(BASE_DIR, "data") # folder with all data station
DIR_OUT = os.path.join(BASE_DIR, "out") # folder with all data station
NETWORK = "ES__ASOS" # Change
START = datetime.datetime(2011, 1, 1) # 1 janvier 2001
END = datetime.datetime(2013, 12, 31) # 31 décembre 2010
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment