...
 
Commits (4)
......@@ -9,9 +9,12 @@ Generate a nice transcript of records automatically for you 👌
## Setting up the project:
Here, we will be using `venv` to manage the environnement and packages. But you can use what you want.
- Create a new virtual environement using under the `venv` directory:
- Install Chrome or Chromium
- Create a new virtual environment for python 3 using under the `venv` directory:
> ⚠ Python 2.7 is not supported and won't be supported.
```bash
$ cd Enhanced-Transcript/
......@@ -27,17 +30,16 @@ $ pip install -r requirements.txt
- Download `chromedriver` (it is needed by `selenium` to scrap data):
- If on Linux or OSX, you can use your package manager for that.
- Alternatively, you can do it manually: put it in the `venv/bin/` folder. . On Linux:
- Alternatively, you can [do it manually](](http://chromedriver.chromium.org/downloads)): put it in the `venv/bin/` folder. . On Linux:
```bash
wget https://chromedriver.storage.googleapis.com/<version_number>/chromedriver_<your_os>.zip
unzip chromedriver_<your_os>.zip
mv chromedriver $(pipenv --venv)/bin
FOLDER=$(dirname `which python`)
mv chromedriver $FOLDER
rm chromedriver_<your_os>.zip
```
## Usage
### As a student from the University of Technology of Compiègne 🍺
......@@ -83,7 +85,6 @@ This project is organised in (simplistic) modules:
- `settings.py`: various settings about paths
## Data Schema
Data gets persisted in a local SQLite database under this schema.
......
......@@ -22,7 +22,10 @@ if __name__ == "__main__":
student_details = get_student_details()
# Cooking the transcript 👨‍🍳
latexer.generate_transcript(student_details, session,
remove_temp_dir=False,
add_sem_comment=False,
break_page_on_semester=True)
latexer.generate_transcript(
student_details,
session,
remove_temp_dir=False,
add_sem_comment=False,
break_page_on_semester=True,
)
#!/usr/bin/env python3
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.models import reset_db, Course
from src.utc_specific.scrapper import UTCScrapper
from src.settings import DATA_FOLDER
from src.utc_specific.scrapper import JSDumpUTCScrapper
from src.utils import get_db_string
if __name__ == "__main__":
scrapper = UTCScrapper(results_file=os.path.join(DATA_FOLDER, 'results.html'))
scrapper = JSDumpUTCScrapper()
# Cleaning things
reset_db()
......
......@@ -21,7 +21,7 @@ class DriverWrapper:
:param to_find: the text to find
"""
el = self._d.find_element_by_id(drop_down)
for option in el.find_elements_by_tag_name('option'):
for option in el.find_elements_by_tag_name("option"):
if option.text == to_find:
option.click()
break
......
......@@ -19,17 +19,27 @@ def translate_missing_info(session_maker, source="fr", target="en", key_file=".k
# NOTE — about `list(filter(.,.)`\bellow : query.filter(.) should work using several conditions
# TODO : resolve this using query.filter(. and .) insted
session = session_maker()
missing_transl = session.query(CourseDescription) \
.filter(CourseDescription.curriculum == "").all()
missing_transl = (
session.query(CourseDescription)
.filter(CourseDescription.curriculum == "")
.all()
)
missing_transl = list(filter(lambda cr: cr.lang == target, missing_transl))
missing_transl = sorted(missing_transl, key=lambda cd: cd.course_code)
associated_codes = list(map(lambda course_desc: course_desc.course_code, missing_transl))
associated_codes = list(
map(lambda course_desc: course_desc.course_code, missing_transl)
)
with_transl = session.query(CourseDescription) \
.filter(CourseDescription.course_code.in_(associated_codes)).all()
with_transl = (
session.query(CourseDescription)
.filter(CourseDescription.course_code.in_(associated_codes))
.all()
)
with_transl = list(filter(lambda cr: cr.lang == source and cr.curriculum != "", with_transl))
with_transl = list(
filter(lambda cr: cr.lang == source and cr.curriculum != "", with_transl)
)
with_transl = sorted(with_transl, key=lambda cd: cd.course_code)
......@@ -42,18 +52,18 @@ def translate_missing_info(session_maker, source="fr", target="en", key_file=".k
content = file.readline()
# Fix some fuzzy formatting
key = content.replace("\"", "").replace("\n", "")
key = content.replace('"', "").replace("\n", "")
if key == "":
raise Exception(f"Key missing in the file {key_file}")
raise Exception("Key missing in the file {}".format(key_file))
for to_translate in with_transl:
data = {
"text": to_translate.curriculum,
"format": "plain",
"lang": f"{source}-{target}",
"key": key
"lang": "{}-{}".format(source, target),
"key": key,
}
res = requests.post(url, data)
......@@ -62,7 +72,7 @@ def translate_missing_info(session_maker, source="fr", target="en", key_file=".k
except KeyError:
print(res.status_code)
print(res.content)
print(f"> {to_translate.course_code}")
print("> {}".format(to_translate.course_code))
print(translation)
print()
code_to_missing[to_translate.course_code].curriculum = translation
......
......@@ -18,19 +18,29 @@ class LaTeXer:
the header, the beginning of the transcript as well as its footer.
"""
def __init__(self, lang: str, final_doc: str,
out_dir=os.path.join(PROJECT_FOLDER, "out"),
templates_dir=os.path.join(PROJECT_FOLDER, "templates")):
def __init__(
self,
lang: str,
final_doc: str,
out_dir=os.path.join(PROJECT_FOLDER, "out"),
templates_dir=os.path.join(PROJECT_FOLDER, "templates"),
):
self._final_doc = final_doc
self._out_dir = out_dir
self._main_doc = os.path.join(out_dir, "main.tex")
self._lang = lang
# Factorization of code: programmatically setting attributes
for field in ["_preamble", "_beg_doc", "_end_doc",
"_semester_template", "_course_template",
"_semester_opts_template", "_diploma_template"]:
file = f"{field[1:]}.tex"
for field in [
"_preamble",
"_beg_doc",
"_end_doc",
"_semester_template",
"_course_template",
"_semester_opts_template",
"_diploma_template",
]:
file = "{}.tex".format(field[1:])
with open(os.path.join(templates_dir, self._lang, file), "r") as tex_file:
setattr(self, field, tex_file.read())
......@@ -56,7 +66,9 @@ class LaTeXer:
return string
def _new_command(self, name, value):
return self._inject_in_string("\\newcommand{\\NAME}{VALUE}", to_inject={"NAME": name, "VALUE": value})
return self._inject_in_string(
"\\newcommand{\\NAME}{VALUE}", to_inject={"NAME": name, "VALUE": value}
)
def _latexify_student_details(self, student_details: StudentDetails):
"""
......@@ -65,10 +77,17 @@ class LaTeXer:
:param student_details:
:return: LaTeXified lines of StudentDetails
"""
return [self._new_command(name.replace("_", ""), getattr(student_details, name)) for name in
get_public_attributes_names(student_details)]
def _course_string(self, course: Course, course_description: CourseDescription, result: CourseResult):
return [
self._new_command(name.replace("_", ""), getattr(student_details, name))
for name in get_public_attributes_names(student_details)
]
def _course_string(
self,
course: Course,
course_description: CourseDescription,
result: CourseResult,
):
"""
Returns a formatted string in LaTeX of the courses information, results
and description.
......@@ -85,17 +104,14 @@ class LaTeXer:
"COURSE_ECTS": course.ects,
"GRADE_OBTAINED": result.result,
"COURSE_OVERVIEW": course_description.overview,
"COURSE_PROGRAM": course_description.curriculum
"COURSE_PROGRAM": course_description.curriculum,
}
return self._inject_in_string(self._course_template, to_inject)
def _diploma_string(self, diploma: Diploma):
to_inject = {
"DIPLOMA_NAME": diploma.name,
"DIPLOMA_PERIOD": diploma.period
}
to_inject = {"DIPLOMA_NAME": diploma.name, "DIPLOMA_PERIOD": diploma.period}
return self._inject_in_string(self._diploma_template, to_inject)
......@@ -113,19 +129,27 @@ class LaTeXer:
formatted_string = self._inject_in_string(self._semester_template, to_inject)
if add_sem_comment:
formatted_string += self._inject_in_string(self._semester_opts_template,
{
"SEMESTER_OBSERVATION": sem.observation
})
formatted_string += self._inject_in_string(
self._semester_opts_template, {"SEMESTER_OBSERVATION": sem.observation}
)
return formatted_string
def _compile_tex_to_pdf(self):
os.system(f"pdflatex -halt-on-error -output-directory {self._out_dir} {self._main_doc}")
def generate_transcript(self, student_details: StudentDetails, session,
remove_temp_dir=True, add_sem_comment=False,
break_page_on_semester=False):
os.system(
"pdflatex -halt-on-error -output-directory {} {}".format(
self._out_dir, self._main_doc
)
)
def generate_transcript(
self,
student_details: StudentDetails,
session,
remove_temp_dir=True,
add_sem_comment=False,
break_page_on_semester=False,
):
"""
Creates the final .tex file
......@@ -148,8 +172,12 @@ class LaTeXer:
latex_student_details = self._latexify_student_details(student_details)
# Adding total number of credits
total_nb_credits = session.query(func.sum(Course.ects)).join(Course.results).all()[0][0]
latex_student_details.append(self._new_command("totalcredits", str(total_nb_credits)))
total_nb_credits = (
session.query(func.sum(Course.ects)).join(Course.results).all()[0][0]
)
latex_student_details.append(
self._new_command("totalcredits", str(total_nb_credits))
)
out.writelines("%s\n" % info for info in latex_student_details)
# Appending the header
......@@ -162,31 +190,47 @@ class LaTeXer:
out.write(self._diploma_string(diploma))
semesters = session.query(Semester) \
.filter(Semester.diploma_short_name == diploma.short_name)
semesters = session.query(Semester).filter(
Semester.diploma_short_name == diploma.short_name
)
for semester in semesters:
out.write(self._semester_string(semester, add_sem_comment))
course_results = session.query(CourseResult) \
.filter(CourseResult.semester_name == semester.name)
course_results = session.query(CourseResult).filter(
CourseResult.semester_name == semester.name
)
for result in course_results:
course_q = session.query(Course) \
.filter(Course.code == result.course_code).all()
description_q = session.query(CourseDescription) \
.filter(CourseDescription.lang == self._lang) \
.filter(CourseDescription.course_code == result.course_code).all()
course_q = (
session.query(Course)
.filter(Course.code == result.course_code)
.all()
)
description_q = (
session.query(CourseDescription)
.filter(CourseDescription.lang == self._lang)
.filter(CourseDescription.course_code == result.course_code)
.all()
)
# Soft replacement if nothing is found for the course or its description
description = CourseDescription() if len(description_q) == 0 else description_q[0]
course = Course(code=description.course_code,
ects=None,
type=None) if len(course_q) == 0 else course_q[0]
course_info_latex = self._course_string(course, description, result).splitlines()
description = (
CourseDescription()
if len(description_q) == 0
else description_q[0]
)
course = (
Course(code=description.course_code, ects=None, type=None)
if len(course_q) == 0
else course_q[0]
)
course_info_latex = self._course_string(
course, description, result
).splitlines()
# Injecting the courses info
out.writelines("%s\n" % l for l in course_info_latex)
......@@ -208,4 +252,4 @@ class LaTeXer:
os.remove(os.path.join(self._out_dir, file))
os.rmdir(self._out_dir)
print(f"Transcript available: {self._final_doc}")
print("Transcript available: {}".format(self._final_doc))
import textwrap
from sqlalchemy import create_engine, Column, Integer, String, Text, Boolean, ForeignKey, Date
from sqlalchemy import (
create_engine,
Column,
Integer,
String,
Text,
Boolean,
ForeignKey,
Date,
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
......@@ -10,7 +19,7 @@ Base = declarative_base()
class Diploma(Base):
__tablename__ = 'diploma'
__tablename__ = "diploma"
short_name = Column(Text, primary_key=True)
......@@ -26,15 +35,17 @@ class Diploma(Base):
@hybrid_property
def period(self):
return f'{self.start.strftime("%B %Y")} -- {self.end.strftime("%B %Y")}'
return "{} -- {}".format(
self.start.strftime("%B %Y"), self.end.strftime("%B %Y")
)
class Semester(Base):
__tablename__ = 'semester'
__tablename__ = "semester"
name = Column(String, primary_key=True)
diploma_short_name = Column(String, ForeignKey('diploma.short_name'))
diploma_short_name = Column(String, ForeignKey("diploma.short_name"))
start = Column(Date)
end = Column(Date)
......@@ -45,7 +56,9 @@ class Semester(Base):
@hybrid_property
def period(self):
return f"{self.first_month} {self.year} -- {self.last_month} {self.year_end_semester}"
return "{} {} -- {} {}".format(
self.first_month, self.year, self.last_month, self.year_end_semester
)
@hybrid_property
def is_fall(self):
......@@ -57,11 +70,19 @@ class Semester(Base):
@hybrid_property
def first_month(self):
return self.start.strftime("%B").replace("February", "Février").replace("September", "Septembre")
return (
self.start.strftime("%B")
.replace("February", "Février")
.replace("September", "Septembre")
)
@hybrid_property
def last_month(self):
return self.end.strftime("%B").replace("January", "Janvier").replace("July", "Juillet")
return (
self.end.strftime("%B")
.replace("January", "Janvier")
.replace("July", "Juillet")
)
@hybrid_property
def code(self):
......@@ -76,11 +97,11 @@ class Semester(Base):
return self.end.strftime("%Y")
def __repr__(self):
return f"Semester {self.name} : {self.level}"
return "Semester {} : {}".format(self.name, self.level)
class Course(Base):
__tablename__ = 'course'
__tablename__ = "course"
code = Column(String, primary_key=True)
ects = Column(Integer)
......@@ -90,8 +111,8 @@ class Course(Base):
has_final_exam = Column(Boolean)
def __repr__(self):
repr = f"{self.code} ({self.type})\n"
repr += f" - worths {self.ects} ECTS\n"
repr = "{} ({})\n".format(self.code, self.type)
repr += " - worths {} ECTS\n".format(self.ects)
repr += " - taught in Spring\n" if self.taught_in_spring else ""
repr += " - taught in Fall\n" if self.taught_in_fall else ""
repr += " - has a final exam\n" if self.has_final_exam else ""
......@@ -102,7 +123,7 @@ class Course(Base):
class CourseDescription(Base):
__tablename__ = "course_description"
course_code = Column(String, ForeignKey('course.code'), primary_key=True)
course_code = Column(String, ForeignKey("course.code"), primary_key=True)
lang = Column(String, primary_key=True)
title = Column(String)
......@@ -123,43 +144,54 @@ class CourseDescription(Base):
course = relationship("Course", back_populates="descriptions")
def __repr__(self):
def format_paragraph(repr):
return "\n".join(textwrap.wrap(repr))
repr = f"Description of {self.course_code} in {self.lang}\n\n"
repr += f" - Title: {format_paragraph(self.title)}\n"
repr += f" - Overview:\n{format_paragraph(self.overview)}\n\n"
repr += f" - Curriculum:\n {format_paragraph(self.curriculum)}\n\n"
repr += f" - Outcomes:\n {format_paragraph(self.outcomes)}\n\n"
repr += f" - Training Objectives:\n {format_paragraph(self.training_objectives)}\n\n"
repr += f" - Pedagogical Objectives:\n {format_paragraph(self.pedagogical_objectives)}\n\n"
repr += f" - Other Objectives:\n {format_paragraph(self.other_objectives)}\n\n"
repr += f" - Bibliography:\n {format_paragraph(self.bibliography)}\n\n"
repr += f" - Recommended level:\n {format_paragraph(self.recommended_level)}\n\n"
repr += f" - Assessment Criteria:\n {format_paragraph(self.assessment_criteria)}\n\n"
repr += f" - Success Criteria:\n {format_paragraph(self.success_criteria)}\n\n"
repr += f" - Misc.:\n {format_paragraph(self.misc)}\n"
repr = "Description of {} in {}\n\n".format(self.course_code, self.lang)
repr += " - Title: {}\n".format(format_paragraph(self.title))
repr += " - Overview:\n{}\n\n".format(format_paragraph(self.overview))
repr += " - Curriculum:\n {}\n\n".format(format_paragraph(self.curriculum))
repr += " - Outcomes:\n {}\n\n".format(format_paragraph(self.outcomes))
repr += " - Training Objectives:\n {}\n\n".format(
format_paragraph(self.training_objectives)
)
repr += " - Pedagogical Objectives:\n {}\n\n".format(
format_paragraph(self.pedagogical_objectives)
)
repr += " - Other Objectives:\n {}\n\n".format(
format_paragraph(self.other_objectives)
)
repr += " - Bibliography:\n {}\n\n".format(format_paragraph(self.bibliography))
repr += " - Recommended level:\n {}\n\n".format(
format_paragraph(self.recommended_level)
)
repr += " - Assessment Criteria:\n {}\n\n".format(
format_paragraph(self.assessment_criteria)
)
repr += " - Success Criteria:\n {}\n\n".format(
format_paragraph(self.success_criteria)
)
repr += " - Misc.:\n {}\n".format(format_paragraph(self.misc))
repr += "\n"
return repr
class CourseResult(Base):
__tablename__ = 'course_result'
__tablename__ = "course_result"
course_code = Column(String, ForeignKey('course.code'), primary_key=True)
semester_name = Column(String, ForeignKey('semester.name'), primary_key=True)
course_code = Column(String, ForeignKey("course.code"), primary_key=True)
semester_name = Column(String, ForeignKey("semester.name"), primary_key=True)
result = Column(String)
course = relationship("Course", back_populates="results")
semester = relationship("Semester", back_populates="results")
def __repr__(self):
return f"{self.course_code}@{self.semester_name}: {self.result}"
return "{}@{}: {}".format(self.course_code, self.semester_name, self.result)
# Foreign Keys Management
......@@ -168,13 +200,19 @@ class CourseResult(Base):
Diploma.semesters = relationship("Semester", back_populates="diploma")
# CourseDescription.course_code => Course.code
Course.descriptions = relationship("CourseDescription", order_by=Course.code, back_populates="course")
Course.descriptions = relationship(
"CourseDescription", order_by=Course.code, back_populates="course"
)
# CourseResult.course_code => Course.code
Course.results = relationship("CourseResult", order_by=Course.code, back_populates="course")
Course.results = relationship(
"CourseResult", order_by=Course.code, back_populates="course"
)
# CourseResult.semester_name => Semester.name
Semester.results = relationship("CourseResult", order_by=Semester.name, back_populates="semester")
Semester.results = relationship(
"CourseResult", order_by=Semester.name, back_populates="semester"
)
def create_tables(verbose=False):
......
import os
PROJECT_FOLDER = os.path.abspath(os.path.join(os.path.realpath(__file__), os.pardir, os.pardir))
PROJECT_FOLDER = os.path.abspath(
os.path.join(os.path.realpath(__file__), os.pardir, os.pardir)
)
# Cached data for personal results and courses information and descriptions
DATA_FOLDER = os.path.join(PROJECT_FOLDER, "data") # not sync with .git by default
STUDENT_DETAILS_FILE = os.path.join(DATA_FOLDER, 'student_details.pickle')
STUDENT_DETAILS_FILE = os.path.join(DATA_FOLDER, "student_details.pickle")
# The local data base persisting the data
LOCAL_DATABASE = os.path.join(DATA_FOLDER, 'courses_and_results.db')
LOCAL_DATABASE = os.path.join(DATA_FOLDER, "courses_and_results.db")
CREDENTIALS_FILE = os.path.join(PROJECT_FOLDER, ".credentials")
DB_URL_FILE = os.path.join(PROJECT_FOLDER, ".db_url")
......
......@@ -68,18 +68,20 @@ class UTCXMLParser:
associated_text = entry.next_sibling.next_sibling.text
course_descr_content[field] = associated_text
except AttributeError as e:
print(f"{type(e)} thrown while parsing")
print(f" - UV: {code}")
print(f" - Entry: {entry}")
print(f" - Field: {field}")
print("{} thrown while parsing".format(type(e)))
print(" - UV: {}".format(code))
print(" - Entry: {}".format(entry))
print(" - Field: {}".format(field))
print(e)
course = Course(code=code,
type=type_,
ects=ects,
taught_in_fall=taught_in_fall,
taught_in_spring=taught_in_spring,
has_final_exam=has_final_exam)
course = Course(
code=code,
type=type_,
ects=ects,
taught_in_fall=taught_in_fall,
taught_in_spring=taught_in_spring,
has_final_exam=has_final_exam,
)
course_descr = CourseDescription(**course_descr_content)
......@@ -87,7 +89,6 @@ class UTCXMLParser:
class FrenchParser(UTCXMLParser):
def __init__(self):
self.lang = "fr"
......@@ -113,12 +114,11 @@ class FrenchParser(UTCXMLParser):
"pedagogical_objectives": "Objectifs pédagogiques spécifiques : ",
"other_objectives": "Objectifs pédagogiques transverses : ",
"curriculum": "Programme : ",
"outcomes": "Résultats : "
"outcomes": "Résultats : ",
}
class EnglishParsen(UTCXMLParser):
class EnglishParser(UTCXMLParser):
def __init__(self):
self.lang = "en"
......@@ -144,5 +144,5 @@ class EnglishParsen(UTCXMLParser):
"pedagogical_objectives": "Specific pedagogical objectives : ",
"other_objectives": "Transversal specific objectives : ",
"curriculum": "Program : ",
"outcomes": "Results : "
"outcomes": "Results : ",
}
This diff is collapsed.
......@@ -15,8 +15,11 @@ def get_public_attributes_names(obj, include_methods_name=False):
:param include_methods_name : If True, include the methods names in the list.
:return: a list of string containing the attributes' names.
"""
def condition_on_methods(name):
return include_methods_name or not (isinstance(getattr(obj, name), types.MethodType))
return include_methods_name or not (
isinstance(getattr(obj, name), types.MethodType)
)
return [name for name in dir(obj) if name[0] != "_" and condition_on_methods(name)]
......@@ -38,7 +41,9 @@ def build_object(obj):
for attr_name in attr_names:
default_value = getattr(obj, attr_name)
pretty_attr_name = attr_name.replace("_", " ").capitalize()
new_value = input(f"{pretty_attr_name} (current value : {default_value}) : ")
new_value = input(
"{} (current value : {}) : ".format(pretty_attr_name, default_value)
)
if new_value != "":
setattr(obj, attr_name, new_value)
return obj
......@@ -60,14 +65,14 @@ def get_db_string(force_local=False, database_url_file: str = DB_URL_FILE):
print("Using local database (forced)")
return db_string
if not(os.path.exists(database_url_file)):
if not (os.path.exists(database_url_file)):
print("Using local SQLite database : " + db_string)
return db_string
file = open(database_url_file, "r")
content = file.readline()
file.close()
parsed = content.replace("\"", "").replace("\n", "")
parsed = content.replace('"', "").replace("\n", "")
if len(parsed) == 0:
print("Using local SQLite database : " + db_string)
else:
......
......@@ -2,16 +2,15 @@ import unittest
class TestStringMethods(unittest.TestCase):
def test_upper(self):
self.assertEqual('foo'.upper(), 'FOO')
self.assertEqual("foo".upper(), "FOO")
def test_issupper(self):
self.assertTrue('FOO'.isupper())
self.assertFalse('Foo'.isupper())
self.assertTrue("FOO".isupper())
self.assertFalse("Foo".isupper())
def test_split(self):
s = 'hello world'
s = "hello world"
self.assertEqual(s.split(), ["hello", "world"])
with self.assertRaises(TypeError):
......