insert_country.py 3.48 KB
Newer Older
1
2
3
4
5
6
7
8
#!/usr/bin/env python3

"""
Script to insert the country data in the database

IT HAS TO BE RUN INSIDE ./manage.py shell
"""
import os
Florent Chehab's avatar
Florent Chehab committed
9
import argparse
Florent Chehab's avatar
Florent Chehab committed
10
import pandas as pd
Florent Chehab's avatar
Florent Chehab committed
11
import requests
12

Florent Chehab's avatar
Florent Chehab committed
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
if __name__ != "__main__":
    print(__name__)
    raise Exception("Script has to be run directly")

# getting args from command line
parser = argparse.ArgumentParser(
    description='Country and Region Insertion')

parser.add_argument('--address', '-a', dest='address',
                    action='store', default='http://127.0.0.1:8000/api',
                    help='Adresse of the web REST API')

parser.add_argument('--token', '-t', dest='token',
                    action='store', default='NO_TOKEN_PROVIDED',
                    help='Permanent token to use')

args = parser.parse_args()

api_address = str(args.address)
api_token = str(args.token)
33

Florent Chehab's avatar
Florent Chehab committed
34
tmp = os.path.join(os.path.realpath(__file__), '../../assets/country.csv')
35
country_file_loc = os.path.abspath(tmp)
36
37
38
tmp = os.path.join(os.path.realpath(__file__),
                   '../../assets/alpha-conv-table.csv')
conv_alpha_file_loc = os.path.abspath(tmp)
39

Florent Chehab's avatar
Florent Chehab committed
40
41
42
43
44
45
46
47
data = pd.read_csv(country_file_loc, sep=',', header=0,
                   dtype=object).fillna('')
data = data.drop(["Least Developed Countries (LDC)",
                  "Land Locked Developing Countries (LLDC)",
                  "Small Island Developing States (SIDS)",
                  "Developed / Developing Countries"],
                 axis=1)

48
49
50
51
52
53
54
55
# Need to load the information for converting
# Countries alpha-3 code to alpha-2 code
data_conv = pd.read_csv(conv_alpha_file_loc, sep=',',
                        header=0, na_filter=False)
conv_alpha_code = {}
for index, row in data_conv.iterrows():
    conv_alpha_code[row["alpha-3"]] = row["alpha-2"]

Florent Chehab's avatar
Florent Chehab committed
56
57
58
59
60
61
62
63
64
65
66
67
68
69

def make_post(address, data):
    h = {'Authorization': 'Token ' + api_token}
    r = requests.post(address, headers=h, data=data)
    if r.status_code is 403:
        raise Exception("Authentification failed")
    if not r.ok:
        print(r.content)
        print(data)
    return r.ok


def save_country(name, code, region):
    a = api_address + "/country/"
70
    data = {'name': name, 'iso_alpha2_code': code, 'region': region}
Florent Chehab's avatar
Florent Chehab committed
71
72
73
74
75
76
77
78
79
    return make_post(a, data)


def save_region(name, code, parent):
    a = api_address + "/region/"
    data = {'name': name, 'un_code': code, 'parent': parent}
    return make_post(a, data)


Florent Chehab's avatar
Florent Chehab committed
80
81
82
83
84
r = [("Region Code", "Region Name"),
     ("Sub-region Code", "Sub-region Name"),
     ("Intermediate Region Code", "Intermediate Region Name"),
     ("ISO-alpha3 Code", "Country or Area")]

Florent Chehab's avatar
Florent Chehab committed
85
inserted_regions = []
Florent Chehab's avatar
Florent Chehab committed
86
87

for index, row in data.iterrows():
Florent Chehab's avatar
Florent Chehab committed
88
    last_region_code = None
Florent Chehab's avatar
Florent Chehab committed
89
90
91
92
93
    for i in range(len(r) - 1):
        r_type = r[i]
        region_code = row[r_type[0]]
        region_name = row[r_type[1]]
        if region_code is not '':
Florent Chehab's avatar
Florent Chehab committed
94
            last_region_code = region_code
Florent Chehab's avatar
Florent Chehab committed
95
96
            if region_code in inserted_regions:
                continue
Florent Chehab's avatar
Florent Chehab committed
97
98
99
100
101
            # look for parent region
            parent = None
            if i > 0:
                parent_code = row[r[i - 1][0]]
                if parent_code is not '':
Florent Chehab's avatar
Florent Chehab committed
102
103
104
                    parent = parent_code
            save_region(region_name, region_code, parent)
            inserted_regions.append(region_code)
Florent Chehab's avatar
Florent Chehab committed
105
106
        else:
            break
107

Florent Chehab's avatar
Florent Chehab committed
108
109
    c_code = row[r[3][0]]
    c_name = row[r[3][1]]
110
111
112
113
114
    if str(c_code) in conv_alpha_code.keys():
        c_code = conv_alpha_code[str(c_code)]
        save_country(c_name, c_code, last_region_code)
    else:
        print("ignoring country :", c_name)