Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Thomas Meurou
NF26 Project
Commits
f0029c0c
Commit
f0029c0c
authored
Jun 20, 2019
by
Unknown
Browse files
Little change
parent
a929da85
Changes
1
Hide whitespace changes
Inline
Side-by-side
question2.py
View file @
f0029c0c
...
@@ -2,167 +2,9 @@ import numpy as np
...
@@ -2,167 +2,9 @@ import numpy as np
import
matplotlib.pyplot
as
plt
import
matplotlib.pyplot
as
plt
from
functools
import
reduce
from
functools
import
reduce
from
database_pre2
import
connection
from
database_pre2
import
connection
import
matplotlib.pyplot
as
plt
import
re
import
re
import
folium
import
folium
def
add
(
x
,
y
):
return
x
+
y
def
abs_diff
(
x
,
y
):
return
abs
(
x
-
y
)
def
diff
(
x
,
y
):
return
x
-
y
#caculate mean reduce
#input [count,mean]
def
reduceFonction
(
x
,
y
):
result
=
[]
for
i
in
range
(
2
):
result
.
append
(
reduce
(
add
,[
x
[
i
],
y
[
i
]]))
return
result
#input [valeur] -> [count,mean]
def
mapFonction1
(
x
):
return
[
1
,
x
]
#input [count,mean] -> [mean]
def
mapFonction2
(
x
):
return
x
[
1
]
/
x
[
0
]
def
testNan
(
x
):
test
=
x
!=
x
return
test
def
mapReduce_kmeans
(
data
,
targetNB
):
results
=
dict
()
for
row
in
data
.
result
():
data_target
=
row
[
targetNB
]
if
testNan
(
data_target
):
continue
data_espace
=
(
row
[
1
],
row
[
2
],
row
[
3
])
if
results
.
get
(
data_espace
)
is
None
:
results
[
data_espace
]
=
mapFonction1
(
data_target
)
else
:
mapresult
=
mapFonction1
(
data_target
)
results
[
data_espace
]
=
reduceFonction
(
mapresult
,
results
[
data_espace
])
for
eachEspace
in
results
:
results
[
eachEspace
]
=
mapFonction2
(
results
[
eachEspace
])
return
results
def
cluster_nb_diff
(
centre_new
,
centre
):
sum
=
0
for
i
in
range
(
3
):
sum
+=
abs
(
centre_new
[
i
][
0
]
-
centre
[
i
][
0
])
return
sum
/
3
#input [tmpt] -> [tmpt,tmpt,tmpt,tmpt]
def
map1_kmeans
(
x
):
return
[
x
,
x
,
x
,
x
]
def
mapCentre
(
x
):
return
[
x
[
0
],
x
[
1
],
x
[
2
],
0
]
#input [tmpt,tmpt,tmpt,tmpt] and [c1,c2,c3,0] -> [|tmpt - c1|,|tmpt - c2|,|tmpt - c3|,tmpt]
def
reduceKmeans
(
x
,
y
):
result
=
[]
for
i
in
range
(
4
):
result
.
append
(
reduce
(
abs_diff
,[
x
[
i
],
y
[
i
]]))
return
result
#input [|tmpt - c1|,|tmpt - c2|,|tmpt - c3|,tmpt] -> [cluster number, min(|tmpt - c|), tmpt]
def
map2_kmeans
(
x
):
min_value
=
10000000000000
index
=
0
for
each
in
range
(
3
):
if
min_value
>
x
[
each
]:
min_value
=
x
[
each
]
index
=
each
return
[
index
,
min_value
,
x
[
3
]]
def
MapnewCentre
(
x
):
return
x
[
1
]
/
x
[
0
]
def
kmeans
(
data
,
targetNB
):
#3centre with [point count, temprature centre]
centre
=
{
0
:[
1
,
0
],
1
:[
1
,
0
],
2
:[
1
,
0
]}
#cluster est pour stocler lat, lon de chaque point de chaque cluster
cluster
=
[[],[],[]]
result
=
mapReduce_kmeans
(
data
,
targetNB
)
#mettre le premier 3 point comme le centres init
init_point_values
=
[
result
[
i
]
for
i
in
result
.
keys
()][:
3
]
init_point_keys
=
[
i
for
i
in
result
.
keys
()][:
3
]
for
key
in
centre
.
keys
():
centre
[
key
]
=
[
1
,
init_point_values
[
key
]]
cluster
[
key
].
append
(
init_point_keys
[
key
])
#init the centre new and result new for mapreduce
centre_new
=
{
0
:[
0
,
0
],
1
:[
0
,
0
],
2
:[
0
,
0
]}
result_new
=
dict
()
#When the number of point of cluster don't change,stop
while
True
:
for
eachkey
in
result
:
if
eachkey
in
cluster
[
0
]
or
eachkey
in
cluster
[
1
]
or
eachkey
in
cluster
[
2
]:
continue
#caculate the distance between the data of this lingne and the centre
#Map1_kemeans
result_new
[
eachkey
]
=
map1_kmeans
(
result
[
eachkey
])
centre_values
=
[]
for
each
in
centre
:
centre_values
.
append
(
centre
[
each
][
1
])
centre_values
=
mapCentre
(
centre_values
)
#Reduce
result_new
[
eachkey
]
=
reduceKmeans
(
result_new
[
eachkey
],
centre_values
)
#Map2_kmeans
result_new
[
eachkey
]
=
map2_kmeans
(
result_new
[
eachkey
])
#Put all the distance and points into the clusters
#Result format [cluster number, min(|tmpt - c|),tmpt - c]
for
eachpoint
in
result_new
:
clusterNB
=
result_new
[
eachpoint
][
0
]
centre_new
[
clusterNB
][
0
]
+=
1
centre_new
[
clusterNB
][
1
]
+=
result_new
[
eachpoint
][
2
]
cluster
[
clusterNB
].
append
(
eachpoint
)
#compare centre_new and centre, if
if
not
cluster_nb_diff
(
centre_new
,
centre
)
>
1
:
break
else
:
#caculate the new centre
print
(
'jasdlkjalsdkjalskd '
,
cluster_nb_diff
(
centre_new
,
centre
))
for
eachculster
in
centre_new
:
centre_new
[
eachculster
][
1
]
=
MapnewCentre
(
centre_new
[
eachculster
])
centre
=
centre_new
centre_new
=
{
0
:[
0
,
0
],
1
:[
0
,
0
],
2
:[
0
,
0
]}
result_new
=
dict
()
cluster
=
[[],[],[]]
createMap
(
cluster
)
def
createMap
(
data
):
def
createMap
(
data
):
mean_lat
=
0
mean_lat
=
0
...
@@ -179,27 +21,30 @@ def createMap(data):
...
@@ -179,27 +21,30 @@ def createMap(data):
mean_lat
=
mean_lat
/
count
mean_lat
=
mean_lat
/
count
mean_lon
=
mean_lon
/
count
mean_lon
=
mean_lon
/
count
m
=
folium
.
Map
(
location
=
[
mean_lon
,
mean_lat
],
zoom_start
=
6
)
m
=
folium
.
Map
(
location
=
[
mean_lon
,
mean_lat
],
zoom_start
=
6
)
# Attributes names which will be displayed on the map
color
=
{
0
:
'blue'
,
1
:
'red'
,
2
:
'green'
}
attributes
=
[
"alti"
,
"drct"
,
"dwpf"
,
"feel"
,
"gust"
,
"ice_accretion_1hr"
,
"ice_accretion_3hr"
,
"ice_accretion_6hr"
,
attributes
=
[
"alti"
,
"drct"
,
"dwpf"
,
"feel"
,
"gust"
,
"ice_accretion_1hr"
,
"ice_accretion_3hr"
,
"ice_accretion_6hr"
,
"metar"
,
"mslp"
,
"p01i"
,
"peak_wind_drct"
,
"peak_wind_gust"
,
"peak_wind_time"
,
"relh"
,
"sknt"
,
"metar"
,
"mslp"
,
"p01i"
,
"peak_wind_drct"
,
"peak_wind_gust"
,
"peak_wind_time"
,
"relh"
,
"sknt"
,
"skyc1"
,
"skyc2"
,
"skyc3"
,
"skyc4"
,
"skyl1"
,
"skyl2"
,
"skyl3"
,
"skyl4"
,
"tmpf"
,
"vsby"
,
"wxcodes"
]
"skyc1"
,
"skyc2"
,
"skyc3"
,
"skyc4"
,
"skyl1"
,
"skyl2"
,
"skyl3"
,
"skyl4"
,
"tmpf"
,
"vsby"
,
"wxcodes"
]
for
each
in
data
.
result
():
for
each
in
data
.
result
():
# print(each)
# print(each)
# Here we choose not to display the "nan" values and the METAR ID
l
=
[
attributes
[
i
]
+
":"
+
str
(
each
[
i
+
4
])
for
i
in
range
(
len
(
attributes
))
if
str
(
each
[
i
+
4
])
!=
'nan'
l
=
[
attributes
[
i
]
+
":"
+
str
(
each
[
i
+
4
])
for
i
in
range
(
len
(
attributes
))
if
str
(
each
[
i
+
4
])
!=
'nan'
and
attributes
[
i
]
!=
"metar"
]
and
attributes
[
i
]
!=
"metar"
]
string
=
'
\n
'
.
join
(
l
)
string
=
'
\n
'
.
join
(
l
)
folium
.
Marker
([
each
[
2
],
each
[
1
]],
folium
.
Marker
([
each
[
2
],
each
[
1
]],
popup
=
string
,
popup
=
string
,
icon
=
folium
.
Icon
(
color
=
'red'
)).
add_to
(
m
)
icon
=
folium
.
Icon
(
color
=
'red'
)).
add_to
(
m
)
m
.
save
(
"Projet-NF26/map.html"
)
m
.
save
(
"Projet-NF26/map.html"
)
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
session
=
connection
()
session
=
connection
()
timestamp
=
'2017-12-02 00:30:00'
# Timestamp user wants to search
date
=
input
(
"Please enter the date you want to search (format 'yyyy-MM-dd'):"
)
time
=
input
(
"Please enter the time you want to search (format 'hh:MM:ss'):"
)
#timestamp = '2017-12-02 00:30:00'
timestamp
=
date
+
' '
+
time
data
=
session
.
execute_async
(
"select * from meurouth_cql.database_time where date = '%s' ALLOW FILTERING"
%
(
timestamp
))
data
=
session
.
execute_async
(
"select * from meurouth_cql.database_time where date = '%s' ALLOW FILTERING"
%
(
timestamp
))
createMap
(
data
)
createMap
(
data
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment