Loading function

loading.py

+from cassandra.cluster import Cluster
+import csv
+import re
+
+# Country: Finland
+# Dates : 2001 to 2010
+
+## datas format:
+# station: three or four character site identifier
+# valid: timestamp of the observation
+# tmpf: Air Temperature in Fahrenheit, typically @ 2 meters
+# dwpf: Dew Point Temperature in Fahrenheit, typically @ 2 meters
+# relh: Relative Humidity in %
+# drct: Wind Direction in degrees from north
+# sknt: Wind Speed in knots
+# p01i: One hour precipitation for the period from the observation time to the time of the previous hourly precipitation reset. This varies slightly by site. Values are in inches. This value may or may not contain frozen precipitation melted by some device on the sensor or estimated by some other means. Unfortunately, we do not know of an authoritative database denoting which station has which sensor.
+# alti: Pressure altimeter in inches
+# mslp: Sea Level Pressure in millibar
+# vsby: Visibility in miles
+# gust: Wind Gust in knots
+# skyc1: Sky Level 1 Coverage
+# skyc2: Sky Level 2 Coverage
+# skyc3: Sky Level 3 Coverage
+# skyc4: Sky Level 4 Coverage
+# skyl1: Sky Level 1 Altitude in feet
+# skyl2: Sky Level 2 Altitude in feet
+# skyl3: Sky Level 3 Altitude in feet
+# skyl4: Sky Level 4 Altitude in feet
+# wxcodes: Present Weather Codes (space seperated)
+# feel: Apparent Temperature (Wind Chill or Heat Index) in Fahrenheit
+# ice_accretion_1hr: Ice Accretion over 1 Hour (inches)
+# ice_accretion_3hr: Ice Accretion over 3 Hours (inches)
+# ice_accretion_6hr: Ice Accretion over 6 Hours (inches)
+# peak_wind_gust: Peak Wind Gust (from PK WND METAR remark) (knots)
+# peak_wind_drct: Peak Wind Gust Direction (from PK WND METAR remark) (deg)
+# peak_wind_time: Peak Wind Gust Time (from PK WND METAR remark)
+# metar: unprocessed reported observation in METAR format
+
+def loadata(filename):
+	dateparser = re.compile(
+		"(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+) (?P<hour>\d+):(?P<minute>\d+)"
+	)
+	with open(filename) as f:
+		for r in csv.DictReader(f):
+			data = {}
+			data["station"] = r["station"]
+			valid = dateparser.match(r["valid"]).groupdict()
+			data["date_obs"] = (
+				int(valid["year"]),
+				int(valid["month"]),
+				int(valid["day"]),
+			)
+			data["time_obs"] = (
+				int(valid["hour"]),
+				int(valid["minute"]),
+			)
+			data["lon"] = float(r["lon"])
+			data["lat"] = float(r["lat"])
+			data["tmpf"] = float(r["tmpf"])
+			data["dwpf"] = float(r["dwpf"])
+			data["relh"] = float(r["relh"])
+			data["drct"] = float(r["drct"])
+			data["sknt"] = float(r["sknt"])
+			data["p01i"] = float(r["p01i"])
+			data["alti"] = float(r["alti"])
+			data["mslp"] = float(r["mslp"])
+			data["vsby"] = float(r["vsby"])
+			data["gust"] = float(r["gust"])
+			data["skyc1"] = float(r["skyc1"])
+			data["skyc2"] = float(r["skyc2"])
+			data["skyc3"] = float(r["skyc3"])
+			data["skyc4"] = float(r["skyc4"])
+			data["skyl1"] = float(r["skyl1"])
+			data["skyl2"] = float(r["skyl2"])
+			data["skyl3"] = float(r["skyl3"])
+			data["skyl4"] = float(r["skyl4"])
+			data["wxcodes"] = r["wxcodes"]
+			data["feel"] = float(r["feel"])
+			data["ice_accretion_1hr"] = float(r["ice_accretion_1hr"])
+			data["ice_accretion_3hr"] = float(r["ice_accretion_3hr"])
+			data["ice_accretion_6hr"] = float(r["ice_accretion_6hr"])
+			data["peak_wind_gust"] = float(r["peak_wind_gust"])
+			data["peak_wind_drct"] = float(r["peak_wind_drct"])
+			data["peak_wind_time"] = float(r["peak_wind_time"])
+			data["metar"] = r["metar"]
+			
+			yield data