utils.py 8.38 KB
Newer Older
1 2 3 4 5 6 7
import logging
import os

import requests

from backend_app.models.course import Course
from backend_app.models.currency import Currency
8
from backend_app.models.exchange import Exchange, update_denormalized_univ_major_minor
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
from backend_app.models.offer import Offer
from backend_app.models.partner import Partner
from external_data.models import ExternalDataUpdateInfo

logger = logging.getLogger("django")

FETCHED_PAGE_SIZE = 500


def paged_data_walk(url, callback):
    """
    Helper function to fetch and parse paginated data from a paginated endpoint
    :param url: base endpoint location
    :param callback: function executed with the content of paginated response
    """
    current_page = 0
    while True:
        response = requests.get(
            "{}?page={}&size={}".format(url, current_page, FETCHED_PAGE_SIZE)
        )
        response_data = response.json()
        logger.info(
            "handling page {} out of {}".format(
                response_data["number"] + 1, response_data["totalPages"]
            )
        )
        callback(response_data["content"])
        if response_data["last"]:
            break
        else:
            current_page += 1


class FixerData(object):
    FIXER_API_TOKEN = os.environ["FIXER_API_TOKEN"].strip()

45
    def update(self):
46 47
        logger.info("Updating currencies from Fixer API")
        response = requests.get(
48
            "http://data.fixer.io/api/latest?access_key={}".format(self.FIXER_API_TOKEN)
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
        )

        data = response.json()

        if data["success"]:
            for (code, rate) in response.json()["rates"].items():
                Currency.objects.update_or_create(
                    code=code, defaults={"one_EUR_in_this_currency": rate}
                )
            ExternalDataUpdateInfo.objects.create(source="fixer")
            logger.info("Currency update was successful")

        else:
            logger.error(
                "Updating currency information from fixer failed, see response from the API below."
            )
            logger.error(response.json())


class UtcData(object):
    UTC_API_ENDPOINT = os.environ["UTC_API_ENDPOINT"].strip()

71 72 73 74 75
    def __init__(self):
        self.all_exchanges_utc_id_from_ent = set()
        self.all_courses_utc_id_from_ent = set()

    def update(self):
76
        logger.info("Updating UTC Data")
77
        self.update_partners()
78
        paged_data_walk(
79 80
            "{}/openedDestinations".format(self.UTC_API_ENDPOINT),
            self.__import_opened_destinations,
81 82
        )
        paged_data_walk(
83
            "{}/exchanges".format(self.UTC_API_ENDPOINT), self.__import_exchange_data
84 85
        )
        paged_data_walk(
86
            "{}/courses".format(self.UTC_API_ENDPOINT), self.__import_course_data
87 88
        )
        ExternalDataUpdateInfo.objects.create(source="utc")
89 90
        logger.info("Updating invalidated")
        self.__update_invalidated()
91 92
        logger.info("Updating UTC info done !")

93 94
        update_denormalized_univ_major_minor()

95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
    def __update_invalidated(self):
        """
        Function to update the unlinked status of exchanges and courses.
        It must be run after update.
        """
        for exchange in Exchange.objects.all():
            if exchange.utc_id not in self.all_exchanges_utc_id_from_ent:
                exchange.unlinked = True
                exchange.save()

        for course in Course.objects.all():
            if course.utc_id not in self.all_courses_utc_id_from_ent:
                course.unlinked = True
                course.save()

    def update_partners(self):
        paged_data_walk(
            "{}/universities".format(self.UTC_API_ENDPOINT),
            self.__import_utc_partner_data,
        )

        if Partner.objects.filter(university=None).count() > 0:
            logger.warning(
                "There are UTC partners that are not mapped to REX-DRI university"
            )

    @classmethod
    def get_local_partners(cls):
        return set(map(lambda p: p.pk, Partner.objects.all()))

    def update_one_student(self, student_utc_login):
        local_partners = self.get_local_partners()

        response_exchange = requests.get(
            "{}/exchanges/{}".format(self.UTC_API_ENDPOINT, student_utc_login)
        ).json()

        for exchange in response_exchange:
            exchange_id = exchange["idDepart"]
            partner_id = exchange["idEtab"]
            if partner_id not in local_partners:
                # An update of the partners in REX-DRI db is also needed
                self.update_partners()
                local_partners = self.get_local_partners()
                if partner_id not in local_partners:
                    raise Exception(
                        "Unknown partner for an exchange {}".format(exchange_id)
                    )

        self.__import_exchange_data(response_exchange)

        response_courses = requests.get(
            "{}/courses/{}".format(self.UTC_API_ENDPOINT, student_utc_login)
        ).json()
        self.__import_course_data(response_courses)

151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
    @classmethod
    def __import_utc_partner_data(cls, response_content):
        logger.info("Importing university partners")

        for univ in response_content:
            logger.debug(univ["idEtab"])

            Partner.objects.update_or_create(
                utc_id=univ["idEtab"],
                defaults=dict(
                    univ_name=univ["nomEtab"],
                    address1=univ["adr1"],
                    address2=univ["adr2"],
                    zipcode=univ["codePostal"],
                    city=univ["ville"],
                    country=univ["pays"],
                    iso_code=univ["codeIso"],
                ),
            )

171
    def __import_exchange_data(self, response_content):
172 173 174 175
        logger.info("Importing exchange data")

        for exchange in response_content:
            exchange_id = exchange["idDepart"]
176
            self.all_exchanges_utc_id_from_ent.add(exchange_id)
177 178 179 180 181 182 183 184 185 186 187
            logger.debug("Exchange id: {}".format(exchange_id))

            Exchange.objects.update_or_create(
                utc_id=exchange_id,
                defaults=dict(
                    utc_partner_id=exchange["idEtab"],
                    semester=exchange["semestreDepart"][0],
                    year=exchange["semestreDepart"][1:],
                    duration=exchange["duree"],
                    double_degree=exchange["doubleDiplome"],
                    master_obtained=exchange["master"],
188
                    student_major_and_semester=exchange["specialite"],
189 190 191
                    student_minor=exchange["option"],
                    utc_allow_courses=exchange["autorisationTransfertUv"],
                    utc_allow_login=exchange["autorisationTransfertLogin"],
192
                    unlinked=False,
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
                ),
            )

    @classmethod
    def __import_opened_destinations(cls, response_content):
        logger.info("Importing the list of opened destinations")

        for destination in response_content:
            logger.info(destination["idEtab"])

            semester = destination["semestre"][:1]
            year = destination["semestre"][1:]

            Offer.objects.update_or_create(
                utc_partner_id=destination["idEtab"],
                year=year,
                semester=semester,
                defaults=dict(
                    comment=destination["commentaire"],
                    nb_seats_offered=destination["nombrePlaces"],
                    double_degree=destination["doubleDiplome"],
                    is_master_offered=destination["master"],
                    specialties=destination["branches"],
                ),
            )

219
    def __import_course_data(self, response_content):
220 221 222
        logger.info("Importing courses data")

        for course in response_content:
223 224
            course_utc_id = course["idCours"]
            self.all_courses_utc_id_from_ent.add(course_utc_id)
225
            Course.objects.update_or_create(
226
                utc_id=course_utc_id,
227 228 229 230 231 232 233 234 235 236
                defaults=dict(
                    utc_exchange_id=course["idDepart"],
                    code=course["code"],
                    title=course["titre"],
                    link=course["lien"],
                    ects=course["ects"],
                    category=course["categorie"],
                    profile=course["profil"],
                    tsh_profile=course["categorieTsh"],
                    student_login=course["login"],
237
                    unlinked=False,
238 239
                ),
            )