Feat: get_exams for CSVLoader

This commit is contained in:
Bertrand Benjamin 2021-04-05 21:21:45 +02:00
parent c1fd060707
commit bd91bf51d6
8 changed files with 145 additions and 106 deletions

View File

@ -0,0 +1,5 @@
Trimestre,Nom,Date,Exercice,Question,Competence,Domaine,Commentaire,Bareme,Est_nivele,Star Tice,Umberto Dingate,Starlin Crangle,Humbert Bourcq,Gabriella Handyside,Stewart Eaves,Erick Going,Ase Praton,Rollins Planks,Dunstan Sarjant,Stacy Guiton,Ange Stanes,Amabelle Elleton,Darn Broomhall,Dyan Chatto,Keane Rennebach,Nari Paulton,Brandy Wase,Jaclyn Firidolfi,Violette Lockney
1,DS,12/01/2021,Exercice 1,1,Calculer,Plop,Coucou,1,1,,,1.0,0,1.0,2.0,3.0,0.0,3.0,3.0,2.0,,1.0,,,,,,,
1,DS,12/01/2021,Exercice 1,2,Calculer,C'est trop chouette!,Coucou,1,1,,,1.0,2,,,3.0,3.0,,,,,2.0,,,,,,,
1,DS,12/01/2021,Exercice 1,3,Calculer,Null,Coucou,1,1,,,,3,2.0,,,,,,,,3.0,,,,,,,
1,DS,12/01/2021,Exercice 1,3,Calculer,Nié,DChic,1,1,,,,2,,,,,,,,,,,,,,,,
1 Trimestre Nom Date Exercice Question Competence Domaine Commentaire Bareme Est_nivele Star Tice Umberto Dingate Starlin Crangle Humbert Bourcq Gabriella Handyside Stewart Eaves Erick Going Ase Praton Rollins Planks Dunstan Sarjant Stacy Guiton Ange Stanes Amabelle Elleton Darn Broomhall Dyan Chatto Keane Rennebach Nari Paulton Brandy Wase Jaclyn Firidolfi Violette Lockney
2 1 DS 12/01/2021 Exercice 1 1 Calculer Plop Coucou 1 1 1.0 0 1.0 2.0 3.0 0.0 3.0 3.0 2.0 1.0
3 1 DS 12/01/2021 Exercice 1 2 Calculer C'est trop chouette! Coucou 1 1 1.0 2 3.0 3.0 2.0
4 1 DS 12/01/2021 Exercice 1 3 Calculer Null Coucou 1 1 3 2.0 3.0
5 1 DS 12/01/2021 Exercice 1 3 Calculer Nié DChic 1 1 2

View File

@ -3,30 +3,11 @@ source: ./
output: ./ output: ./
templates: templates/ templates: templates/
competences:
Chercher:
name: Chercher
abrv: Cher
Représenter:
name: Représenter
abrv: Rep
Modéliser:
name: Modéliser
abrv: Mod
Raisonner:
name: Raisonner
abrv: Rai
Calculer:
name: Calculer
abrv: Cal
Communiquer:
name: Communiquer
abrv: Com
tribes: tribes:
- name: Tribe1 Tribe1:
name: Tribe1
type: Type1 type: Type1
students: tribe1.csv students: tribe1.csv
- name: Tribe2 Tribe2:
name: Tribe2
students: tribe2.csv students: tribe2.csv

View File

@ -1,4 +0,0 @@
---
source: sheets/
output: reports/
templates: templates/

View File

@ -18,14 +18,21 @@ class Loader(ABC):
"""Load data from source""" """Load data from source"""
def __init__(self, configfile="recoconfig.yml"): CONFIG = {}
def __init__(self, configfile=""):
"""Init loader """Init loader
:param configfile: yaml file with informations on data source :param configfile: yaml file with informations on data source
""" """
with open(CONFIGPATH, "r") as config: self._config = self.CONFIG
sefl._config = yaml.load(config, Loader=yaml.FullLoader) if configfile.endswith(".yml"):
with open(configfile, "r") as config:
self._config.update(yaml.load(config, Loader=yaml.FullLoader))
def get_config(self):
""" Get config"""
return self._config
@abstractmethod @abstractmethod
def get_tribes(self): def get_tribes(self):
@ -77,6 +84,5 @@ class Writer(ABC):
""" Write datas to the source """ """ Write datas to the source """
@abstractmethod
def __init__(self): def __init__(self):
pass pass

View File

@ -0,0 +1,44 @@
---
source: ./ # basepath where to start
competences: # Competences
Chercher:
name: Chercher
abrv: Cher
Représenter:
name: Représenter
abrv: Rep
Modéliser:
name: Modéliser
abrv: Mod
Raisonner:
name: Raisonner
abrv: Rai
Calculer:
name: Calculer
abrv: Cal
Communiquer:
name: Communiquer
abrv: Com
valid_scores: #
BAD: 0 # Everything is bad
FEW: 1 # Few good things
NEARLY: 2 # Nearly good but things are missing
GOOD: 3 # Everything is good
NOTFILLED: # The item is not scored yet
NOANSWER: . # Student gives no answer (count as 0)
ABS: "a" # Student has absent (this score won't be impact the final mark)
csv_fields: # dataframe_field: csv_field
term: Trimestre
exam: Nom
date: Date
exercise: Exercice
question: Question
competence: Competence
theme: Domaine
comment: Commentaire
score_rate: Bareme
is_leveled: Est_nivele

View File

@ -5,11 +5,18 @@ import pandas as pd
from pathlib import Path from pathlib import Path
__all__ = ["list_csvs", "extract_exam"] __all__ = ["list_csvs", "extract_fields"]
def list_csvs(path): def list_csvs(path):
""" list csv files in path """ """list csv files in path
:example:
>>> list_csvs("./example/Tribe1/")
[PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')]
>>> list_csvs("./example/Tribe1")
[PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')]
"""
return list(Path(path).glob("*.csv")) return list(Path(path).glob("*.csv"))

View File

@ -1,78 +1,59 @@
#!/usr/bin/env python #!/usr/bin/env python
# encoding: utf-8 # encoding: utf-8
from .. import Loader
import yaml import yaml
import os
from pathlib import Path
import pandas as pd
from .. import Loader
from .lib import *
def list_csvs(path): DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "default_config.yml")
""" list csv files in path """ with open(DEFAULT_CONFIG_FILE, "r") as config:
pass DEFAULT_CONFIG = yaml.load(config, Loader=yaml.FullLoader)
class CSVLoader(Loader): class CSVLoader(Loader):
"""Loader when scores and metadatas are stored in csv files """Loader when scores and metadatas are stored in csv files
## configfile (`recoconfig.yml` by default :config:
source: ./ # basepath where to start (default value) :example:
templates: # directory where templates are stored >>> loader = CSVLoader()
>>> loader.get_config()
{'source': './', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}}
>>> loader = CSVLoader("./test_config.yml")
>>> loader.get_config()
{'source': './example', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}, 'output': './output', 'templates': 'templates/', 'tribes': {'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}}
tribes: # All the tribes (required)
Tribe1: # Tribe name
directory: tribe1 # tribe directory
type: type1 # Type of tribe (2nd, 1, T...)
students: tribe1.csv # csv with infos on students
competences: # Competences (default values)
Chercher:
name: Chercher
abrv: Cher
Représenter:
name: Représenter
abrv: Rep
Modéliser:
name: Modéliser
abrv: Mod
Raisonner:
name: Raisonner
abrv: Rai
Calculer:
name: Calculer
abrv: Cal
Communiquer:
name: Communiquer
abrv: Com
valid_scores: # (default values)
BAD: 0 # Everything is bad
FEW: 1 # Few good things
NEARLY: 2 # Nearly good but things are missing
GOOD: 3 # Everything is good
NOTFILLED: # The item is not scored yet
NOANSWER: . # Student gives no answer (count as 0)
ABS: "a" # Student has absent (this score won't be impact the final mark)
csv_fields: # dataframe_field: csv_field (default values)
term: Trimestre,
exam: Nom,
date: Date,
exercise: Exercice,
question: Question,
competence: Competence,
theme: Domaine,
comment: Commentaire,
score_rate: Bareme,
is_leveled: Est_nivele,
""" """
CONFIG = DEFAULT_CONFIG
def get_config(self): def get_config(self):
""" Get config""" """ Get config """
return self._config return self._config
def rename_columns(self, dataframe):
"""Rename dataframe column to match with `csv_fields` """
return dataframe.rename(columns=self._config["csv_fields"])
def reverse_csv_field(self, keys):
""" Reverse csv field from keys """
return [self._config["csv_fields"][k] for k in keys]
def get_tribes(self, only_names=False): def get_tribes(self, only_names=False):
""" Get tribes list """ """Get tribes list
:example:
>>> loader = CSVLoader("./test_config.yml")
>>> loader.get_tribes()
{'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}
>>> loader.get_tribes(only_names=True)
['Tribe1', 'Tribe2']
"""
if only_names: if only_names:
return list(self._config["tribes"].keys()) return list(self._config["tribes"].keys())
return self._config["tribes"] return self._config["tribes"]
@ -81,46 +62,52 @@ class CSVLoader(Loader):
"""Get exams list """Get exams list
:param tribes: get only exams for those tribes :param tribes: get only exams for those tribes
:return: list of dictionaries of exams (fields: `["name", "tribe", "date", "term"]) :return: list of dictionaries of exams (fields: `["exam", "tribe", "date", "term", "score_file"]`)
:example:
>>> loader = CSVLoader("./test_config.yml")
>>> loader.get_exams(["Tribe1"])
Nom Date Trimestre score_file tribe
0 DS 12/01/2021 1 example/Tribe1/210112_DS.csv Tribe1
0 DS6 22/01/2021 1 example/Tribe1/210122_DS6.csv Tribe1
""" """
exams = [] exams = []
for tribe in tribes: for tribe in tribes:
csvs = list_csvs() tribe_path = Path(self._config["source"]) / tribe
csvs = list_csvs(tribe_path)
for csv in csvs: for csv in csvs:
fields = [ fields = self.reverse_csv_field(["exam", "date", "term"])
self._config["csv_fields"][k] for k in ["exam", "date", "term"]
]
exam = extract_fields(csv, fields) exam = extract_fields(csv, fields)
exam.rename(columns=self._config["csv_fields"], inplace=True).rename( exam = self.rename_columns(exam)
columns={"exam": "name"}, inplace=True exam["score_file"] = csv
) exam["tribe"] = tribe
exams.append(exam)
return df.concate(exams) return pd.concat(exams)
def get_students(self, tribes=[]): def get_students(self, tribes=[]):
"""Get student list """Get student list
:param filters: list of filters :param filters: list of filters
""" """
pass return ""
def get_exam_questions(self, exams=[]): def get_exam_questions(self, exams=[]):
"""Get questions for the exam """Get questions for the exam
:param exams: questions for those exams only :param exams: questions for those exams only
""" """
pass return ""
def get_questions_scores(self, questions=[]): def get_questions_scores(self, questions=[]):
"""Get scores of those questions """Get scores of those questions
:param questions: score for those questions :param questions: score for those questions
""" """
pass return ""
def get_student_scores(self, student): def get_student_scores(self, student):
"""Get scores of the student """Get scores of the student
:param student: :param student:
""" """
pass return ""

13
test_config.yml Normal file
View File

@ -0,0 +1,13 @@
---
source: ./example
output: ./output
templates: templates/
tribes:
Tribe1:
name: Tribe1
type: Type1
students: tribe1.csv
Tribe2:
name: Tribe2
students: tribe2.csv