From bd91bf51d64dbf90c7bb118b5cc21c015675fea6 Mon Sep 17 00:00:00 2001 From: Bertrand Benjamin Date: Mon, 5 Apr 2021 21:21:45 +0200 Subject: [PATCH] Feat: get_exams for CSVLoader --- example/Tribe1/210112_DS.csv | 5 + example/recoconfig.yml | 33 +---- recoconfig.yml | 4 - recopytex/store/__init__.py | 16 ++- recopytex/store/filesystem/default_config.yml | 44 ++++++ recopytex/store/filesystem/lib.py | 11 +- recopytex/store/filesystem/loader.py | 125 ++++++++---------- test_config.yml | 13 ++ 8 files changed, 145 insertions(+), 106 deletions(-) create mode 100644 example/Tribe1/210112_DS.csv delete mode 100644 recoconfig.yml create mode 100644 recopytex/store/filesystem/default_config.yml create mode 100644 test_config.yml diff --git a/example/Tribe1/210112_DS.csv b/example/Tribe1/210112_DS.csv new file mode 100644 index 0000000..d518d37 --- /dev/null +++ b/example/Tribe1/210112_DS.csv @@ -0,0 +1,5 @@ +Trimestre,Nom,Date,Exercice,Question,Competence,Domaine,Commentaire,Bareme,Est_nivele,Star Tice,Umberto Dingate,Starlin Crangle,Humbert Bourcq,Gabriella Handyside,Stewart Eaves,Erick Going,Ase Praton,Rollins Planks,Dunstan Sarjant,Stacy Guiton,Ange Stanes,Amabelle Elleton,Darn Broomhall,Dyan Chatto,Keane Rennebach,Nari Paulton,Brandy Wase,Jaclyn Firidolfi,Violette Lockney +1,DS,12/01/2021,Exercice 1,1,Calculer,Plop,Coucou,1,1,,,1.0,0,1.0,2.0,3.0,0.0,3.0,3.0,2.0,,1.0,,,,,,, +1,DS,12/01/2021,Exercice 1,2,Calculer,C'est trop chouette!,Coucou,1,1,,,1.0,2,,,3.0,3.0,,,,,2.0,,,,,,, +1,DS,12/01/2021,Exercice 1,3,Calculer,Null,Coucou,1,1,,,,3,2.0,,,,,,,,3.0,,,,,,, +1,DS,12/01/2021,Exercice 1,3,Calculer,Nié,DChic,1,1,,,,2,,,,,,,,,,,,,,,, diff --git a/example/recoconfig.yml b/example/recoconfig.yml index bf32a2d..30da86a 100644 --- a/example/recoconfig.yml +++ b/example/recoconfig.yml @@ -3,30 +3,11 @@ source: ./ output: ./ templates: templates/ -competences: - Chercher: - name: Chercher - abrv: Cher - Représenter: - name: Représenter - abrv: Rep - Modéliser: - name: Modéliser - abrv: Mod - Raisonner: - name: Raisonner - abrv: Rai - Calculer: - name: Calculer - abrv: Cal - Communiquer: - name: Communiquer - abrv: Com - - tribes: - - name: Tribe1 - type: Type1 - students: tribe1.csv - - name: Tribe2 - students: tribe2.csv + Tribe1: + name: Tribe1 + type: Type1 + students: tribe1.csv + Tribe2: + name: Tribe2 + students: tribe2.csv diff --git a/recoconfig.yml b/recoconfig.yml deleted file mode 100644 index 9a64f7c..0000000 --- a/recoconfig.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -source: sheets/ -output: reports/ -templates: templates/ diff --git a/recopytex/store/__init__.py b/recopytex/store/__init__.py index 206d3d5..b9ed17a 100644 --- a/recopytex/store/__init__.py +++ b/recopytex/store/__init__.py @@ -18,14 +18,21 @@ class Loader(ABC): """Load data from source""" - def __init__(self, configfile="recoconfig.yml"): + CONFIG = {} + + def __init__(self, configfile=""): """Init loader :param configfile: yaml file with informations on data source - """ - with open(CONFIGPATH, "r") as config: - sefl._config = yaml.load(config, Loader=yaml.FullLoader) + self._config = self.CONFIG + if configfile.endswith(".yml"): + with open(configfile, "r") as config: + self._config.update(yaml.load(config, Loader=yaml.FullLoader)) + + def get_config(self): + """ Get config""" + return self._config @abstractmethod def get_tribes(self): @@ -77,6 +84,5 @@ class Writer(ABC): """ Write datas to the source """ - @abstractmethod def __init__(self): pass diff --git a/recopytex/store/filesystem/default_config.yml b/recopytex/store/filesystem/default_config.yml new file mode 100644 index 0000000..54c7787 --- /dev/null +++ b/recopytex/store/filesystem/default_config.yml @@ -0,0 +1,44 @@ +--- +source: ./ # basepath where to start + +competences: # Competences + Chercher: + name: Chercher + abrv: Cher + Représenter: + name: Représenter + abrv: Rep + Modéliser: + name: Modéliser + abrv: Mod + Raisonner: + name: Raisonner + abrv: Rai + Calculer: + name: Calculer + abrv: Cal + Communiquer: + name: Communiquer + abrv: Com + +valid_scores: # + BAD: 0 # Everything is bad + FEW: 1 # Few good things + NEARLY: 2 # Nearly good but things are missing + GOOD: 3 # Everything is good + NOTFILLED: # The item is not scored yet + NOANSWER: . # Student gives no answer (count as 0) + ABS: "a" # Student has absent (this score won't be impact the final mark) + +csv_fields: # dataframe_field: csv_field + term: Trimestre + exam: Nom + date: Date + exercise: Exercice + question: Question + competence: Competence + theme: Domaine + comment: Commentaire + score_rate: Bareme + is_leveled: Est_nivele + diff --git a/recopytex/store/filesystem/lib.py b/recopytex/store/filesystem/lib.py index 8fa2546..079e7db 100644 --- a/recopytex/store/filesystem/lib.py +++ b/recopytex/store/filesystem/lib.py @@ -5,11 +5,18 @@ import pandas as pd from pathlib import Path -__all__ = ["list_csvs", "extract_exam"] +__all__ = ["list_csvs", "extract_fields"] def list_csvs(path): - """ list csv files in path """ + """list csv files in path + + :example: + >>> list_csvs("./example/Tribe1/") + [PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')] + >>> list_csvs("./example/Tribe1") + [PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')] + """ return list(Path(path).glob("*.csv")) diff --git a/recopytex/store/filesystem/loader.py b/recopytex/store/filesystem/loader.py index e389bc7..234faf7 100644 --- a/recopytex/store/filesystem/loader.py +++ b/recopytex/store/filesystem/loader.py @@ -1,78 +1,59 @@ #!/usr/bin/env python # encoding: utf-8 -from .. import Loader import yaml +import os +from pathlib import Path +import pandas as pd +from .. import Loader +from .lib import * -def list_csvs(path): - """ list csv files in path """ - pass +DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "default_config.yml") +with open(DEFAULT_CONFIG_FILE, "r") as config: + DEFAULT_CONFIG = yaml.load(config, Loader=yaml.FullLoader) class CSVLoader(Loader): """Loader when scores and metadatas are stored in csv files - ## configfile (`recoconfig.yml` by default + :config: - source: ./ # basepath where to start (default value) - templates: # directory where templates are stored + :example: + >>> loader = CSVLoader() + >>> loader.get_config() + {'source': './', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}} + >>> loader = CSVLoader("./test_config.yml") + >>> loader.get_config() + {'source': './example', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}, 'output': './output', 'templates': 'templates/', 'tribes': {'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}} - tribes: # All the tribes (required) - Tribe1: # Tribe name - directory: tribe1 # tribe directory - type: type1 # Type of tribe (2nd, 1, T...) - students: tribe1.csv # csv with infos on students - - competences: # Competences (default values) - Chercher: - name: Chercher - abrv: Cher - Représenter: - name: Représenter - abrv: Rep - Modéliser: - name: Modéliser - abrv: Mod - Raisonner: - name: Raisonner - abrv: Rai - Calculer: - name: Calculer - abrv: Cal - Communiquer: - name: Communiquer - abrv: Com - - valid_scores: # (default values) - BAD: 0 # Everything is bad - FEW: 1 # Few good things - NEARLY: 2 # Nearly good but things are missing - GOOD: 3 # Everything is good - NOTFILLED: # The item is not scored yet - NOANSWER: . # Student gives no answer (count as 0) - ABS: "a" # Student has absent (this score won't be impact the final mark) - - csv_fields: # dataframe_field: csv_field (default values) - term: Trimestre, - exam: Nom, - date: Date, - exercise: Exercice, - question: Question, - competence: Competence, - theme: Domaine, - comment: Commentaire, - score_rate: Bareme, - is_leveled: Est_nivele, """ + CONFIG = DEFAULT_CONFIG + def get_config(self): - """ Get config""" + """ Get config """ return self._config + def rename_columns(self, dataframe): + """Rename dataframe column to match with `csv_fields` """ + return dataframe.rename(columns=self._config["csv_fields"]) + + def reverse_csv_field(self, keys): + """ Reverse csv field from keys """ + return [self._config["csv_fields"][k] for k in keys] + def get_tribes(self, only_names=False): - """ Get tribes list """ + """Get tribes list + + :example: + >>> loader = CSVLoader("./test_config.yml") + >>> loader.get_tribes() + {'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}} + >>> loader.get_tribes(only_names=True) + ['Tribe1', 'Tribe2'] + """ if only_names: return list(self._config["tribes"].keys()) return self._config["tribes"] @@ -81,46 +62,52 @@ class CSVLoader(Loader): """Get exams list :param tribes: get only exams for those tribes - :return: list of dictionaries of exams (fields: `["name", "tribe", "date", "term"]) + :return: list of dictionaries of exams (fields: `["exam", "tribe", "date", "term", "score_file"]`) + + :example: + >>> loader = CSVLoader("./test_config.yml") + >>> loader.get_exams(["Tribe1"]) + Nom Date Trimestre score_file tribe + 0 DS 12/01/2021 1 example/Tribe1/210112_DS.csv Tribe1 + 0 DS6 22/01/2021 1 example/Tribe1/210122_DS6.csv Tribe1 """ exams = [] for tribe in tribes: - csvs = list_csvs() + tribe_path = Path(self._config["source"]) / tribe + csvs = list_csvs(tribe_path) for csv in csvs: - fields = [ - self._config["csv_fields"][k] for k in ["exam", "date", "term"] - ] + fields = self.reverse_csv_field(["exam", "date", "term"]) exam = extract_fields(csv, fields) - exam.rename(columns=self._config["csv_fields"], inplace=True).rename( - columns={"exam": "name"}, inplace=True - ) - - return df.concate(exams) + exam = self.rename_columns(exam) + exam["score_file"] = csv + exam["tribe"] = tribe + exams.append(exam) + return pd.concat(exams) def get_students(self, tribes=[]): """Get student list :param filters: list of filters """ - pass + return "" def get_exam_questions(self, exams=[]): """Get questions for the exam :param exams: questions for those exams only """ - pass + return "" def get_questions_scores(self, questions=[]): """Get scores of those questions :param questions: score for those questions """ - pass + return "" def get_student_scores(self, student): """Get scores of the student :param student: """ - pass + return "" diff --git a/test_config.yml b/test_config.yml new file mode 100644 index 0000000..2efa04c --- /dev/null +++ b/test_config.yml @@ -0,0 +1,13 @@ +--- +source: ./example +output: ./output +templates: templates/ + +tribes: + Tribe1: + name: Tribe1 + type: Type1 + students: tribe1.csv + Tribe2: + name: Tribe2 + students: tribe2.csv