Commit 99b84035 authored by Dorian's avatar Dorian

Merge branch 'Location'

parents 705bdfaf 45cabddc
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/NF26 Project.iml" filepath="$PROJECT_DIR$/.idea/NF26 Project.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
import csv
import re
import json
from cassandra.cluster import Cluster
def getData(container, value, type):
result = container[value]
if (result == "M" or result == "T") :
result = None
if (type == "string"):
return result
elif (type == "float") :
if (result != None):
result = float(result)
else :
return -8000
return result
elif (type == "date") :
if (result != None) :
dateparser = re.compile("(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+) (?P<hour>\d+):(?P<minute>\d+)")
match = dateparser.match(container[value])
if not match :
return "continue"
data = match.groupdict()
result = (
int(data["year"]),
int(data["month"]),
int(data["day"]),
int(data["hour"]),
int(data["minute"]),
)
return result
else :
return None
def loadata(filename):
with open(filename) as f:
for r in csv.DictReader(f):
timestamp = getData(r, "valid", "date")
if (timestamp == "continue" or timestamp == None):
continue
data = {}
data["valid"] = timestamp
data["station"] = getData(r, "station", "string")
data["lon"] = getData(r, "lon", "float")
data["lat"] = getData(r, "lat", "float")
def getData(container, value, type):
result = container[value]
if (result == "M" or result == "T") :
result = None
if (type == "string"):
return result
elif (type == "float") :
if (result != None):
result = float(result)
return result
elif (type == "date") :
if (result != None) :
dateparser = re.compile("(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+) (?P<hour>\d+):(?P<minute>\d+)")
match = dateparser.match(container[value])
if not match :
return "continue"
data = match.groupdict()
result = (
int(data["year"]),
int(data["month"]),
int(data["day"]),
int(data["hour"]),
int(data["minute"]),
)
return result
else :
return None
def loadataQ2(filename):
with open(filename) as f:
for r in csv.DictReader(f):
timestamp = getData(r, "valid", "date")
if (timestamp == "continue" or timestamp == None):
continue
data = {}
data["Dhm"] = timestamp
data["Station"] = getData(r, "station", "string")
data["Longitude"] = getData(r, "lon", "float")
data["Latitude"] = getData(r, "lat", "float")
data["Temperature"] = getData(r, "tmpc", "float")
data["DewPoint"] = getData(r, "dwpc", "float")
data["RelativeHumidity"] = getData(r, "relh", "float")
# data["drct"] = getData(r, "drct", "float")
data["WindSpeed"] = getData(r, "sknt", "float")
data["Precipitation"] = getData(r, "p01m", "float")
# data["alti"] = getData(r, "alti", "float")
# data["mslp"] = getData(r, "mslp", "float")
data["Visibility"] = getData(r, "vsby", "float")
# data["gust"] = getData(r, "gust", "float")
# data["skyc1"] = getData(r, "skyc1", "string")
# data["skyc2"] = getData(r, "skyc2", "string")
# data["skyc3"] = getData(r, "skyc3", "string")
# data["skyc4"] = getData(r, "skyc4", "string")
# data["skyl1"] = getData(r, "skyl1", "float")
# data["skyl2"] = getData(r, "skyl2", "float")
# data["skyl3"] = getData(r, "skyl3", "float")
# data["skyl4"] = getData(r, "skyl4", "float")
# data["wxcodes"] = getData(r, "wxcodes", "string")
# data["ice_accretion_1hr"] = getData(r, "ice_accretion_1hr", "float")
# data["ice_accretion_3hr"] = getData(r, "ice_accretion_3hr", "float")
# data["ice_accretion_6hr"] = getData(r, "ice_accretion_6hr", "float")
# data["peak_wind_gust"] = getData(r, "peak_wind_gust", "float")
# data["peak_wind_drct"] = getData(r, "peak_wind_drct", "float")
# data["peak_wind_time"] = getData(r, "peak_wind_time", "float")
data["ApparentTemperature"] = getData(r, "feel", "float")
# data["metar"] = getData(r, "metar", "string")
yield data
def initTableForQ2(session) :
query = '''
DROP TABLE IF EXISTS weatherQ2 ;
'''
session.execute(query)
query = '''
CREATE TABLE weatherQ2(
Station text,
DhmYear varint,
DhmMonth varint,
DhmDay varint,
DhmHour varint,
DhmMinute varint,
Longitude float,
Latitude float,
Temperature float,
DewPoint float,
RelativeHumidity float,
-- drct float,
WindSpeed float,
Precipitation float,
-- alti float,
-- mslp float,
Visibility float,
-- gust float,
-- skyc1 text,
-- skyc2 text,
-- skyc3 text,
-- skyc4 text,
-- skyl1 float,
-- skyl2 float,
-- skyl3 float,
-- skyl4 float,
-- wxcodes text,
-- ice_accretion_1hr float,
-- ice_accretion_3hr float,
-- ice_accretion_6hr float,
-- peak_wind_gust float,
-- peak_wind_drct float,
-- peak_wind_time float,
ApparentTemperature float,
-- metar text,
PRIMARY KEY ((DhmYear, DhmMonth, DhmDay), DhmHour, DhmMinute, Latitude, Longitude)
);
'''
session.execute(query)
def getWeatherQ2(csvfilename, limit):
cluster = Cluster(['localhost'])
session = cluster.connect('dbermond_projet')
initTableForQ2(session)
data = loadataQ2(csvfilename)
n = 0
for r in data:
t = (
r["Station"],
r["Dhm"][0],
r["Dhm"][1],
r["Dhm"][2],
r["Dhm"][3],
r["Dhm"][4],
r["Longitude"],
r["Latitude"],
r["Temperature"],
r["DewPoint"],
r["RelativeHumidity"],
# r["drct"],
r["WindSpeed"],
# r["alti"],
# r["mslp"],
r["Precipitation"],
r["Visibility"],
# r["gust"],
# r["skyc1"],
# r["skyc2"],
# r["skyc3"],
# r["skyc4"],
#
# r["skyl1"],
# r["skyl2"],
# r["skyl3"],
# r["skyl4"],
#
# r["wxcodes"],
#
# r["ice_accretion_1hr"],
# r["ice_accretion_3hr"],
# r["ice_accretion_6hr"],
# r["peak_wind_gust"],
# r["peak_wind_drct"],
# r["peak_wind_time"],
r["ApparentTemperature"],
# r["metar"],
)
query = """
INSERT INTO weatherQ2(
Station,
DhmYear,
DhmMonth,
DhmDay,
DhmHour,
DhmMinute,
Longitude,
Latitude,
Temperature,
DewPoint,
RelativeHumidity,
-- drct float,
WindSpeed,
Precipitation,
-- alti float,
-- mslp float,
Visibility,
-- gust float,
-- skyc1 text,
-- skyc2 text,
-- skyc3 text,
-- skyc4 text,
-- skyl1 float,
-- skyl2 float,
-- skyl3 float,
-- skyl4 float,
-- wxcodes text,
-- ice_accretion_1hr float,
-- ice_accretion_3hr float,
-- ice_accretion_6hr float,
-- peak_wind_gust float,
-- peak_wind_drct float,
-- peak_wind_time float,
ApparentTemperature
-- metar
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
#(%s, %d, %d, %d, %d, %d, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %s, %s, %s, %s, %f, %f, %f, %f, %s, )
session.execute(query, t)
n += 1
if (limit != 0 and n >= limit) :
break
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment