Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Romain Creuzenet
nf26_projet
Commits
247ec043
Commit
247ec043
authored
Jun 16, 2019
by
Romain Creuzenet
Browse files
synchronisation & clean server
parent
cb5c576e
Changes
4
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
247ec043
/__pycache__/*
/env_nf26/*
*/.*
/out
/out/*
.idea/workspace.xml
View file @
247ec043
<?xml version="1.0" encoding="UTF-8"?>
<project
version=
"4"
>
<component
name=
"ChangeListManager"
>
<list
default=
"true"
id=
"dec891dc-2fad-4291-af33-64d4fd64029d"
name=
"Default Changelist"
comment=
""
/>
<list
default=
"true"
id=
"dec891dc-2fad-4291-af33-64d4fd64029d"
name=
"Default Changelist"
comment=
""
>
<change
beforePath=
"$PROJECT_DIR$/.gitignore"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/.gitignore"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/.idea/workspace.xml"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/.idea/workspace.xml"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/main.py"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/main.py"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/parameters.py"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/parameters.py"
afterDir=
"false"
/>
</list>
<option
name=
"EXCLUDED_CONVERTED_TO_IGNORED"
value=
"true"
/>
<option
name=
"SHOW_DIALOG"
value=
"false"
/>
<option
name=
"HIGHLIGHT_CONFLICTS"
value=
"true"
/>
...
...
@@ -22,8 +27,8 @@
<file
pinned=
"false"
current-in-tab=
"true"
>
<entry
file=
"file://$PROJECT_DIR$/main.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"
393
"
>
<caret
line=
"
15
5"
column=
"
24
"
selection-start-line=
"
15
5"
selection-start-column=
"
24
"
selection-end-line=
"
15
5"
selection-end-column=
"
24
"
/>
<state
relative-caret-position=
"
589
"
>
<caret
line=
"
20
5"
column=
"
36
"
selection-start-line=
"
20
5"
selection-start-column=
"
36
"
selection-end-line=
"
20
5"
selection-end-column=
"
36
"
/>
<folding>
<element
signature=
"e#45#75#0"
expanded=
"true"
/>
</folding>
...
...
@@ -43,8 +48,8 @@
<file
pinned=
"false"
current-in-tab=
"false"
>
<entry
file=
"file://$PROJECT_DIR$/parameters.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"
-82
"
>
<caret
line=
"2
6"
column=
"19
"
selection-start-line=
"2
6"
selection-start-column=
"19"
selection-end-line=
"26"
selection-end-column=
"19
"
/>
<state
relative-caret-position=
"
513
"
>
<caret
line=
"2
7
"
selection-start-line=
"2
7"
selection-end-line=
"27
"
/>
<folding>
<element
signature=
"e#62#99#0"
expanded=
"true"
/>
</folding>
...
...
@@ -204,6 +209,13 @@
<entry
file=
"file://$PROJECT_DIR$/data/LFOI_2001_2010.csv"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
/>
</entry>
<entry
file=
"file://$PROJECT_DIR$/download_data.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"-804"
>
<caret
line=
"5"
column=
"52"
selection-start-line=
"5"
selection-start-column=
"52"
selection-end-line=
"5"
selection-end-column=
"52"
/>
</state>
</provider>
</entry>
<entry
file=
"file://$PROJECT_DIR$/create_table.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"811"
>
...
...
@@ -213,25 +225,18 @@
</entry>
<entry
file=
"file://$PROJECT_DIR$/parameters.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"
-82
"
>
<caret
line=
"2
6"
column=
"19
"
selection-start-line=
"2
6"
selection-start-column=
"19"
selection-end-line=
"26"
selection-end-column=
"19
"
/>
<state
relative-caret-position=
"
513
"
>
<caret
line=
"2
7
"
selection-start-line=
"2
7"
selection-end-line=
"27
"
/>
<folding>
<element
signature=
"e#62#99#0"
expanded=
"true"
/>
</folding>
</state>
</provider>
</entry>
<entry
file=
"file://$PROJECT_DIR$/download_data.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"-804"
>
<caret
line=
"5"
column=
"52"
selection-start-line=
"5"
selection-start-column=
"52"
selection-end-line=
"5"
selection-end-column=
"52"
/>
</state>
</provider>
</entry>
<entry
file=
"file://$PROJECT_DIR$/main.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"
393
"
>
<caret
line=
"
15
5"
column=
"
24
"
selection-start-line=
"
15
5"
selection-start-column=
"
24
"
selection-end-line=
"
15
5"
selection-end-column=
"
24
"
/>
<state
relative-caret-position=
"
589
"
>
<caret
line=
"
20
5"
column=
"
36
"
selection-start-line=
"
20
5"
selection-start-column=
"
36
"
selection-end-line=
"
20
5"
selection-end-column=
"
36
"
/>
<folding>
<element
signature=
"e#45#75#0"
expanded=
"true"
/>
</folding>
...
...
main.py
View file @
247ec043
"""File to execute to show results"""
# Data
from
parameters
import
SESSION
from
parameters
import
SESSION
,
DIR_OUT
,
START
,
END
# Basic
import
re
import
matplotlib.dates
as
mdates
import
matplotlib.pyplot
as
plt
import
warnings
import
re
import
os
# Stats
import
statsmodels.graphics
as
stm_graphs
import
pandas
as
pd
...
...
@@ -36,6 +37,7 @@ def ask_q(possibilities, text=">>> "):
def
ask_d
(
text
=
">>> "
):
"""Demande une date"""
print
(
"Entrez une date sous la forme YYYY-MM-DD HH:mm"
)
print
(
"Comprise entre {} et {}"
.
format
(
START
.
strftime
(
'%Y-%m-%d'
),
END
.
strftime
(
'%Y-%m-%d'
)))
date_parser
=
re
.
compile
(
r
"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2}) (?P<hour>\d{2}):(?P<minute>\d{2})"
)
match
=
None
...
...
@@ -49,7 +51,7 @@ def ask_d(text=">>> "):
return
result
def
cho
o
se_attr
():
def
chose_attr
():
"""Permet de demander un attribut dans la table"""
# Search element
decision
=
{
...
...
@@ -73,7 +75,11 @@ class Manager:
table
=
None
# table name use by the function
def
run
(
self
):
"""Chose"""
"""Chose objective"""
# Initialisation
os
.
makedirs
(
DIR_OUT
,
exist_ok
=
True
)
# Chose objective
print
(
"Choisissez ce que vous voulez faire"
)
print
(
"
\t
1 - Pour un point donné de l’espace, je veux pouvoir avoir un historique du passé"
)
print
(
"
\t
2 - À un instant donné je veux pouvoir obtenir une carte me représentant n’importe quel indicateur"
)
...
...
@@ -89,6 +95,7 @@ class Manager:
def
historic
(
self
):
self
.
table
=
"TABLE_SPACE"
print
(
"=== Choix 1 : Historique ==="
)
# Search station
stations
=
[]
...
...
@@ -101,10 +108,10 @@ class Manager:
print
()
station
=
ask_q
(
stations
)
attr
=
cho
o
se_attr
()
attr
=
chose_attr
()
ts
=
pd
.
Series
()
query
=
"SELECT time
,{} FROM {} WHERE station={}"
.
format
(
attr
,
self
.
table
,
station
.
__repr__
())
query
=
"SELECT time,
{} FROM {} WHERE station={}"
.
format
(
attr
,
self
.
table
,
station
.
__repr__
())
for
row
in
execute_query
(
query
):
value
=
getattr
(
row
,
attr
)
if
value
is
None
:
...
...
@@ -117,25 +124,29 @@ class Manager:
plt
.
xticks
(
rotation
=
90
)
plt
.
plot
(
ts
,
label
=
attr
)
plt
.
title
(
"Donnees de
"
+
str
(
attr
)
+
"
pour la station :
"
+
station
)
plt
.
title
(
"Donnees de
{}
pour la station :
{}"
.
format
(
attr
,
station
)
)
plt
.
legend
()
plt
.
show
()
plt
.
savefig
(
'./out/graph_'
+
station
+
'_'
+
str
(
attr
)
+
'.png'
)
path
=
os
.
path
.
join
(
DIR_OUT
,
'graph_{}_{}.png'
.
format
(
station
,
attr
))
plt
.
savefig
(
path
)
res
=
stm
.
tsa
.
seasonal_decompose
(
ts
,
freq
=
15
,
extrapolate_trend
=
'freq'
)
res
.
plot
()
plt
.
show
()
plt
.
savefig
(
'./out/decompose_'
+
station
+
'_'
+
str
(
attr
)
+
'.png'
)
path
=
os
.
path
.
join
(
DIR_OUT
,
'decompose_{}_{}.png'
.
format
(
station
,
attr
))
plt
.
savefig
(
path
)
stm_graphs
.
tsaplots
.
plot_acf
(
ts
,
lags
=
30
)
plt
.
show
()
plt
.
savefig
(
'./out/acf_'
+
station
+
'_'
+
str
(
attr
)
+
'.png'
)
path
=
os
.
path
.
join
(
DIR_OUT
,
'acf_{}_{}.png'
.
format
(
station
,
attr
))
plt
.
savefig
(
path
)
def
map
(
self
):
self
.
table
=
"TABLE_TIME"
print
(
"=== Choix 2 : Map ==="
)
date
=
ask_d
()
attr
=
cho
o
se_attr
()
attr
=
chose_attr
()
plt
.
figure
(
figsize
=
(
14
,
14
))
the_map
=
Basemap
(
projection
=
'mill'
,
llcrnrlat
=
25.281898
,
llcrnrlon
=-
18.42
,
urcrnrlat
=
48.08
,
urcrnrlon
=
10.35
,
...
...
@@ -150,41 +161,54 @@ class Manager:
the_map
.
drawmeridians
(
np
.
arange
(
0
,
360
,
30
))
the_map
.
drawparallels
(
np
.
arange
(
-
90
,
90
,
30
))
try
:
query
=
"SELECT station, lon, lat, {} FROM {} WHERE time={}"
.
format
(
attr
,
self
.
table
,
date
)
for
row
in
execute_query
(
query
):
if
getattr
(
row
,
"station"
)
is
None
:
continue
x
,
y
=
the_map
(
row
.
lon
,
row
.
lat
)
value
=
getattr
(
row
,
attr
)
plt
.
plot
(
x
,
y
,
'go'
)
plt
.
annotate
(
round
(
value
,
1
),
(
x
,
y
))
plt
.
title
(
'Map'
)
plt
.
show
()
plt
.
savefig
(
'./out/map.png'
)
except
:
dateb
=
list
(
date
)
dateb
[
3
]
=
0
dateb
[
4
]
=
0
dateb
=
tuple
(
dateb
)
datee
=
list
(
date
)
datee
[
3
]
=
23
datee
[
4
]
=
59
datee
=
tuple
(
dateb
)
print
(
"Seul ces heures sont disponibles pour ce jour"
)
query
=
"SELECT DISTINCT time FROM {} WHERE time>={} AND time <= {} ALLOW FILTERING"
.
format
(
self
.
table
,
dateb
,
datee
)
date_ok
=
False
# The date is valide
query
=
"SELECT station, lon, lat, {} FROM {} WHERE time={}"
.
format
(
attr
,
self
.
table
,
date
)
for
row
in
execute_query
(
query
):
if
getattr
(
row
,
"station"
)
is
None
or
getattr
(
row
,
attr
)
is
None
:
continue
date_ok
=
True
x
,
y
=
the_map
(
getattr
(
row
,
"lon"
),
getattr
(
row
,
"lat"
))
value
=
getattr
(
row
,
attr
)
plt
.
plot
(
x
,
y
,
'go'
)
plt
.
annotate
(
round
(
value
,
1
),
(
x
,
y
))
title
=
"Map {} du {}"
.
format
(
attr
,
datetime
(
*
list
(
date
)).
strftime
(
'%Y-%m-%d %H:%M'
))
plt
.
title
(
title
)
plt
.
show
()
for
elt
in
' :-'
:
title
=
title
.
replace
(
elt
,
'_'
)
path
=
os
.
path
.
join
(
DIR_OUT
,
title
.
lower
()
+
'.png'
)
plt
.
savefig
(
path
)
# If date is wrong, show some dates for this day
if
not
date_ok
:
date_begin
=
list
(
date
)
date_begin
[
3
]
=
date_begin
[
4
]
=
0
# set hours and minutes at 0
date_begin
=
tuple
(
date_begin
)
date_end
=
list
(
date
)
date_end
[
3
]
=
23
date_end
[
4
]
=
59
date_end
=
tuple
(
date_end
)
print
(
"Seules ces heures sont disponibles pour ce jour"
)
query
=
"SELECT DISTINCT time FROM {} WHERE time >= {} AND time <= {} ALLOW FILTERING"
.
format
(
self
.
table
,
date_begin
,
date_end
,
)
for
row
in
execute_query
(
query
):
print
(
getattr
(
row
,
"time"
))
resp
=
list
(
getattr
(
row
,
"time"
))
print
(
str
(
resp
[
3
]).
zfill
(
2
)
+
":"
+
str
(
resp
[
4
]).
zfill
(
2
),
end
=
" - "
)
def
cluster
(
self
):
self
.
table
=
"TABLE_TIME"
print
(
"=== Choix 3 : CLUSTER ==="
)
# Ask Date
print
(
"
Entrez l
a date de départ"
)
print
(
"
L
a date de départ
:
"
)
date_b
=
ask_d
()
print
(
"Entrez la date de fin"
)
print
(
"Entrez la date de fin
sous la forme YYYY-MM-DD HH:mm
"
)
date_e
=
ask_d
()
print
(
"Entrez le nombre de cluster voulus"
)
nb_clust
=
ask_cluster
()
...
...
parameters.py
View file @
247ec043
...
...
@@ -20,10 +20,11 @@ for key in KEY_SPACE:
else
:
break
if
SESSION
is
None
:
raise
NoHostAvailable
(
"Erreur de conne
x
tion à cassandra"
)
raise
NoHostAvailable
(
"Erreur de conne
c
tion à cassandra"
)
BASE_DIR
=
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
))
# path of project
DIR_DATA
=
os
.
path
.
join
(
BASE_DIR
,
"data"
)
# folder with all data station
DIR_OUT
=
os
.
path
.
join
(
BASE_DIR
,
"out"
)
# folder with all data station
NETWORK
=
"ES__ASOS"
# Change
START
=
datetime
.
datetime
(
2011
,
1
,
1
)
# 1 janvier 2001
END
=
datetime
.
datetime
(
2013
,
12
,
31
)
# 31 décembre 2010
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment