lafrite/recopytex
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Feat: get_exams for CSVLoader

Browse Source
This commit is contained in:
Bertrand Benjamin 2021-04-05 21:21:45 +02:00
parent c1fd060707
commit bd91bf51d6
8 changed files with 145 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
example/Tribe1/210112_DS.csv Normal file
View File

@ -0,0 +1,5 @@
Trimestre,Nom,Date,Exercice,Question,Competence,Domaine,Commentaire,Bareme,Est_nivele,Star Tice,Umberto Dingate,Starlin Crangle,Humbert Bourcq,Gabriella Handyside,Stewart Eaves,Erick Going,Ase Praton,Rollins Planks,Dunstan Sarjant,Stacy Guiton,Ange Stanes,Amabelle Elleton,Darn Broomhall,Dyan Chatto,Keane Rennebach,Nari Paulton,Brandy Wase,Jaclyn Firidolfi,Violette Lockney
1,DS,12/01/2021,Exercice 1,1,Calculer,Plop,Coucou,1,1,,,1.0,0,1.0,2.0,3.0,0.0,3.0,3.0,2.0,,1.0,,,,,,,
1,DS,12/01/2021,Exercice 1,2,Calculer,C'est trop chouette!,Coucou,1,1,,,1.0,2,,,3.0,3.0,,,,,2.0,,,,,,,
1,DS,12/01/2021,Exercice 1,3,Calculer,Null,Coucou,1,1,,,,3,2.0,,,,,,,,3.0,,,,,,,
1,DS,12/01/2021,Exercice 1,3,Calculer,Nié,DChic,1,1,,,,2,,,,,,,,,,,,,,,,
1 Trimestre Nom Date Exercice Question Competence Domaine Commentaire Bareme Est_nivele Star Tice Umberto Dingate Starlin Crangle Humbert Bourcq Gabriella Handyside Stewart Eaves Erick Going Ase Praton Rollins Planks Dunstan Sarjant Stacy Guiton Ange Stanes Amabelle Elleton Darn Broomhall Dyan Chatto Keane Rennebach Nari Paulton Brandy Wase Jaclyn Firidolfi Violette Lockney
2 1 DS 12/01/2021 Exercice 1 1 Calculer Plop Coucou 1 1 1.0 0 1.0 2.0 3.0 0.0 3.0 3.0 2.0 1.0
3 1 DS 12/01/2021 Exercice 1 2 Calculer C'est trop chouette! Coucou 1 1 1.0 2 3.0 3.0 2.0
4 1 DS 12/01/2021 Exercice 1 3 Calculer Null Coucou 1 1 3 2.0 3.0
5 1 DS 12/01/2021 Exercice 1 3 Calculer Nié DChic 1 1 2

27
example/recoconfig.yml
View File

@ -3,30 +3,11 @@ source: ./
output: ./
templates: templates/
competences:
Chercher:
name: Chercher
abrv: Cher
Représenter:
name: Représenter
abrv: Rep
Modéliser:
name: Modéliser
abrv: Mod
Raisonner:
name: Raisonner
abrv: Rai
Calculer:
name: Calculer
abrv: Cal
Communiquer:
name: Communiquer
abrv: Com
tribes:
- name: Tribe1
Tribe1:
name: Tribe1
type: Type1
students: tribe1.csv
- name: Tribe2
Tribe2:
name: Tribe2
students: tribe2.csv

4
recoconfig.yml
View File

@ -1,4 +0,0 @@
---
source: sheets/
output: reports/
templates: templates/

16
recopytex/store/__init__.py
View File

@ -18,14 +18,21 @@ class Loader(ABC):
"""Load data from source"""
def __init__(self, configfile="recoconfig.yml"):
CONFIG = {}
def __init__(self, configfile=""):
"""Init loader
:param configfile: yaml file with informations on data source
"""
with open(CONFIGPATH, "r") as config:
sefl._config = yaml.load(config, Loader=yaml.FullLoader)
self._config = self.CONFIG
if configfile.endswith(".yml"):
with open(configfile, "r") as config:
self._config.update(yaml.load(config, Loader=yaml.FullLoader))
def get_config(self):
""" Get config"""
return self._config
@abstractmethod
def get_tribes(self):
@ -77,6 +84,5 @@ class Writer(ABC):
""" Write datas to the source """
@abstractmethod
def __init__(self):
pass

44
recopytex/store/filesystem/default_config.yml Normal file
View File

@ -0,0 +1,44 @@
---
source: ./ # basepath where to start
competences: # Competences
Chercher:
name: Chercher
abrv: Cher
Représenter:
name: Représenter
abrv: Rep
Modéliser:
name: Modéliser
abrv: Mod
Raisonner:
name: Raisonner
abrv: Rai
Calculer:
name: Calculer
abrv: Cal
Communiquer:
name: Communiquer
abrv: Com
valid_scores: #
BAD: 0 # Everything is bad
FEW: 1 # Few good things
NEARLY: 2 # Nearly good but things are missing
GOOD: 3 # Everything is good
NOTFILLED: # The item is not scored yet
NOANSWER: . # Student gives no answer (count as 0)
ABS: "a" # Student has absent (this score won't be impact the final mark)
csv_fields: # dataframe_field: csv_field
term: Trimestre
exam: Nom
date: Date
exercise: Exercice
question: Question
competence: Competence
theme: Domaine
comment: Commentaire
score_rate: Bareme
is_leveled: Est_nivele

11
recopytex/store/filesystem/lib.py
View File

@ -5,11 +5,18 @@ import pandas as pd
from pathlib import Path
__all__ = ["list_csvs", "extract_exam"]
__all__ = ["list_csvs", "extract_fields"]
def list_csvs(path):
""" list csv files in path """
"""list csv files in path
:example:
>>> list_csvs("./example/Tribe1/")
[PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')]
>>> list_csvs("./example/Tribe1")
[PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')]
"""
return list(Path(path).glob("*.csv"))

123
recopytex/store/filesystem/loader.py
View File

@ -1,78 +1,59 @@
#!/usr/bin/env python
# encoding: utf-8
from .. import Loader
import yaml
import os
from pathlib import Path
import pandas as pd
from .. import Loader
from .lib import *
def list_csvs(path):
""" list csv files in path """
pass
DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "default_config.yml")
with open(DEFAULT_CONFIG_FILE, "r") as config:
DEFAULT_CONFIG = yaml.load(config, Loader=yaml.FullLoader)
class CSVLoader(Loader):
"""Loader when scores and metadatas are stored in csv files
## configfile (`recoconfig.yml` by default
:config:
source: ./ # basepath where to start (default value)
templates: # directory where templates are stored
:example:
>>> loader = CSVLoader()
>>> loader.get_config()
{'source': './', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}}
>>> loader = CSVLoader("./test_config.yml")
>>> loader.get_config()
{'source': './example', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}, 'output': './output', 'templates': 'templates/', 'tribes': {'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}}
tribes: # All the tribes (required)
Tribe1: # Tribe name
directory: tribe1 # tribe directory
type: type1 # Type of tribe (2nd, 1, T...)
students: tribe1.csv # csv with infos on students
competences: # Competences (default values)
Chercher:
name: Chercher
abrv: Cher
Représenter:
name: Représenter
abrv: Rep
Modéliser:
name: Modéliser
abrv: Mod
Raisonner:
name: Raisonner
abrv: Rai
Calculer:
name: Calculer
abrv: Cal
Communiquer:
name: Communiquer
abrv: Com
valid_scores: # (default values)
BAD: 0 # Everything is bad
FEW: 1 # Few good things
NEARLY: 2 # Nearly good but things are missing
GOOD: 3 # Everything is good
NOTFILLED: # The item is not scored yet
NOANSWER: . # Student gives no answer (count as 0)
ABS: "a" # Student has absent (this score won't be impact the final mark)
csv_fields: # dataframe_field: csv_field (default values)
term: Trimestre,
exam: Nom,
date: Date,
exercise: Exercice,
question: Question,
competence: Competence,
theme: Domaine,
comment: Commentaire,
score_rate: Bareme,
is_leveled: Est_nivele,
"""
CONFIG = DEFAULT_CONFIG
def get_config(self):
""" Get config """
return self._config
def rename_columns(self, dataframe):
"""Rename dataframe column to match with `csv_fields` """
return dataframe.rename(columns=self._config["csv_fields"])
def reverse_csv_field(self, keys):
""" Reverse csv field from keys """
return [self._config["csv_fields"][k] for k in keys]
def get_tribes(self, only_names=False):
""" Get tribes list """
"""Get tribes list
:example:
>>> loader = CSVLoader("./test_config.yml")
>>> loader.get_tribes()
{'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}
>>> loader.get_tribes(only_names=True)
['Tribe1', 'Tribe2']
"""
if only_names:
return list(self._config["tribes"].keys())
return self._config["tribes"]
@ -81,46 +62,52 @@ class CSVLoader(Loader):
"""Get exams list
:param tribes: get only exams for those tribes
:return: list of dictionaries of exams (fields: `["name", "tribe", "date", "term"])
:return: list of dictionaries of exams (fields: `["exam", "tribe", "date", "term", "score_file"]`)
:example:
>>> loader = CSVLoader("./test_config.yml")
>>> loader.get_exams(["Tribe1"])
Nom Date Trimestre score_file tribe
0 DS 12/01/2021 1 example/Tribe1/210112_DS.csv Tribe1
0 DS6 22/01/2021 1 example/Tribe1/210122_DS6.csv Tribe1
"""
exams = []
for tribe in tribes:
csvs = list_csvs()
tribe_path = Path(self._config["source"]) / tribe
csvs = list_csvs(tribe_path)
for csv in csvs:
fields = [
self._config["csv_fields"][k] for k in ["exam", "date", "term"]
]
fields = self.reverse_csv_field(["exam", "date", "term"])
exam = extract_fields(csv, fields)
exam.rename(columns=self._config["csv_fields"], inplace=True).rename(
columns={"exam": "name"}, inplace=True
)
return df.concate(exams)
exam = self.rename_columns(exam)
exam["score_file"] = csv
exam["tribe"] = tribe
exams.append(exam)
return pd.concat(exams)
def get_students(self, tribes=[]):
"""Get student list
:param filters: list of filters
"""
pass
return ""
def get_exam_questions(self, exams=[]):
"""Get questions for the exam
:param exams: questions for those exams only
"""
pass
return ""
def get_questions_scores(self, questions=[]):
"""Get scores of those questions
:param questions: score for those questions
"""
pass
return ""
def get_student_scores(self, student):
"""Get scores of the student
:param student:
"""
pass
return ""

13
test_config.yml Normal file
View File

@ -0,0 +1,13 @@
---
source: ./example
output: ./output
templates: templates/
tribes:
Tribe1:
name: Tribe1
type: Type1
students: tribe1.csv
Tribe2:
name: Tribe2
students: tribe2.csv