Feat: get_exams for CSVLoader

example/Tribe1/210112_DS.csv

@@ -0,0 +1,5 @@
+Trimestre,Nom,Date,Exercice,Question,Competence,Domaine,Commentaire,Bareme,Est_nivele,Star Tice,Umberto Dingate,Starlin Crangle,Humbert Bourcq,Gabriella Handyside,Stewart Eaves,Erick Going,Ase Praton,Rollins Planks,Dunstan Sarjant,Stacy Guiton,Ange Stanes,Amabelle Elleton,Darn Broomhall,Dyan Chatto,Keane Rennebach,Nari Paulton,Brandy Wase,Jaclyn Firidolfi,Violette Lockney
+1,DS,12/01/2021,Exercice 1,1,Calculer,Plop,Coucou,1,1,,,1.0,0,1.0,2.0,3.0,0.0,3.0,3.0,2.0,,1.0,,,,,,,
+1,DS,12/01/2021,Exercice 1,2,Calculer,C'est trop chouette!,Coucou,1,1,,,1.0,2,,,3.0,3.0,,,,,2.0,,,,,,,
+1,DS,12/01/2021,Exercice 1,3,Calculer,Null,Coucou,1,1,,,,3,2.0,,,,,,,,3.0,,,,,,,
+1,DS,12/01/2021,Exercice 1,3,Calculer,Nié,DChic,1,1,,,,2,,,,,,,,,,,,,,,,

example/recoconfig.yml

@@ -3,30 +3,11 @@ source: ./
 output: ./
 templates: templates/
 
-competences:
-  Chercher:
-    name: Chercher
-    abrv: Cher
-  Représenter:
-    name: Représenter
-    abrv: Rep
-  Modéliser:
-    name: Modéliser
-    abrv: Mod
-  Raisonner:
-    name: Raisonner
-    abrv: Rai
-  Calculer:
-    name: Calculer
-    abrv: Cal
-  Communiquer:
-    name: Communiquer
-    abrv: Com
-
-
 tribes:
-  - name: Tribe1
+    Tribe1:
-    type: Type1
+        name: Tribe1
-    students: tribe1.csv
+        type: Type1
-  - name: Tribe2
+        students: tribe1.csv
-    students: tribe2.csv
+    Tribe2:
+        name: Tribe2
+        students: tribe2.csv

recoconfig.yml

@@ -1,4 +0,0 @@
----
-source: sheets/
-output: reports/
-templates: templates/

recopytex/store/__init__.py

@@ -18,14 +18,21 @@ class Loader(ABC):
 
     """Load data from source"""
 
-    def __init__(self, configfile="recoconfig.yml"):
+    CONFIG = {}
+
+    def __init__(self, configfile=""):
         """Init loader
 
         :param configfile: yaml file with informations on data source
-
         """
-        with open(CONFIGPATH, "r") as config:
+        self._config = self.CONFIG
-            sefl._config = yaml.load(config, Loader=yaml.FullLoader)
+        if configfile.endswith(".yml"):
+            with open(configfile, "r") as config:
+                self._config.update(yaml.load(config, Loader=yaml.FullLoader))
+
+    def get_config(self):
+        """ Get config"""
+        return self._config
 
     @abstractmethod
     def get_tribes(self):
@@ -77,6 +84,5 @@ class Writer(ABC):
 
     """ Write datas to the source """
 
-    @abstractmethod
     def __init__(self):
         pass

recopytex/store/filesystem/default_config.yml

@@ -0,0 +1,44 @@
+---
+source: ./ # basepath where to start 
+
+competences: # Competences 
+  Chercher:
+    name: Chercher
+    abrv: Cher
+  Représenter:
+    name: Représenter
+    abrv: Rep
+  Modéliser:
+    name: Modéliser
+    abrv: Mod
+  Raisonner:
+    name: Raisonner
+    abrv: Rai
+  Calculer:
+    name: Calculer
+    abrv: Cal
+  Communiquer:
+    name: Communiquer
+    abrv: Com
+
+valid_scores: # 
+  BAD: 0 # Everything is bad
+  FEW: 1 # Few good things
+  NEARLY: 2 # Nearly good but things are missing
+  GOOD: 3 # Everything is good
+  NOTFILLED:   # The item is not scored yet
+  NOANSWER: .  # Student gives no answer (count as 0)
+  ABS: "a"  # Student has absent (this score won't be impact the final mark)
+
+csv_fields: # dataframe_field: csv_field 
+  term: Trimestre
+  exam: Nom
+  date: Date
+  exercise: Exercice
+  question: Question
+  competence: Competence
+  theme: Domaine
+  comment: Commentaire
+  score_rate: Bareme
+  is_leveled: Est_nivele
+

recopytex/store/filesystem/lib.py

@@ -5,11 +5,18 @@ import pandas as pd
 from pathlib import Path
 
 
-__all__ = ["list_csvs", "extract_exam"]
+__all__ = ["list_csvs", "extract_fields"]
 
 
 def list_csvs(path):
-    """ list csv files in path """
+    """list csv files in path
+
+    :example:
+    >>> list_csvs("./example/Tribe1/")
+    [PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')]
+    >>> list_csvs("./example/Tribe1")
+    [PosixPath('example/Tribe1/210112_DS.csv'), PosixPath('example/Tribe1/210122_DS6.csv')]
+    """
     return list(Path(path).glob("*.csv"))
 
 

recopytex/store/filesystem/loader.py

@@ -1,78 +1,59 @@
 #!/usr/bin/env python
 # encoding: utf-8
 
-from .. import Loader
 import yaml
+import os
+from pathlib import Path
+import pandas as pd
+from .. import Loader
+from .lib import *
 
 
-def list_csvs(path):
+DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "default_config.yml")
-    """ list csv files in path """
+with open(DEFAULT_CONFIG_FILE, "r") as config:
-    pass
+    DEFAULT_CONFIG = yaml.load(config, Loader=yaml.FullLoader)
 
 
 class CSVLoader(Loader):
 
     """Loader when scores and metadatas are stored in csv files
 
-    ## configfile (`recoconfig.yml` by default
+    :config:
 
-        source: ./ # basepath where to start (default value)
+    :example:
-        templates: # directory where templates are stored
+    >>> loader = CSVLoader()
+    >>> loader.get_config()
+    {'source': './', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}}
+    >>> loader = CSVLoader("./test_config.yml")
+    >>> loader.get_config()
+    {'source': './example', 'competences': {'Chercher': {'name': 'Chercher', 'abrv': 'Cher'}, 'Représenter': {'name': 'Représenter', 'abrv': 'Rep'}, 'Modéliser': {'name': 'Modéliser', 'abrv': 'Mod'}, 'Raisonner': {'name': 'Raisonner', 'abrv': 'Rai'}, 'Calculer': {'name': 'Calculer', 'abrv': 'Cal'}, 'Communiquer': {'name': 'Communiquer', 'abrv': 'Com'}}, 'valid_scores': {'BAD': 0, 'FEW': 1, 'NEARLY': 2, 'GOOD': 3, 'NOTFILLED': None, 'NOANSWER': '.', 'ABS': 'a'}, 'csv_fields': {'term': 'Trimestre', 'exam': 'Nom', 'date': 'Date', 'exercise': 'Exercice', 'question': 'Question', 'competence': 'Competence', 'theme': 'Domaine', 'comment': 'Commentaire', 'score_rate': 'Bareme', 'is_leveled': 'Est_nivele'}, 'output': './output', 'templates': 'templates/', 'tribes': {'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}}
 
-        tribes: # All the tribes (required)
-          Tribe1: # Tribe name
-            directory: tribe1  # tribe directory
-            type: type1 # Type of tribe (2nd, 1, T...)
-            students: tribe1.csv # csv with infos on students
-
-        competences: # Competences (default values)
-          Chercher:
-            name: Chercher
-            abrv: Cher
-          Représenter:
-            name: Représenter
-            abrv: Rep
-          Modéliser:
-            name: Modéliser
-            abrv: Mod
-          Raisonner:
-            name: Raisonner
-            abrv: Rai
-          Calculer:
-            name: Calculer
-            abrv: Cal
-          Communiquer:
-            name: Communiquer
-            abrv: Com
-
-        valid_scores: # (default values)
-          BAD: 0 # Everything is bad
-          FEW: 1 # Few good things
-          NEARLY: 2 # Nearly good but things are missing
-          GOOD: 3 # Everything is good
-          NOTFILLED:   # The item is not scored yet
-          NOANSWER: .  # Student gives no answer (count as 0)
-          ABS: "a"  # Student has absent (this score won't be impact the final mark)
-
-        csv_fields: # dataframe_field: csv_field (default values)
-          term: Trimestre,
-          exam: Nom,
-          date: Date,
-          exercise: Exercice,
-          question: Question,
-          competence: Competence,
-          theme: Domaine,
-          comment: Commentaire,
-          score_rate: Bareme,
-          is_leveled: Est_nivele,
     """
 
+    CONFIG = DEFAULT_CONFIG
+
     def get_config(self):
-        """ Get config"""
+        """ Get config """
         return self._config
 
+    def rename_columns(self, dataframe):
+        """Rename dataframe column to match with `csv_fields` """
+        return dataframe.rename(columns=self._config["csv_fields"])
+
+    def reverse_csv_field(self, keys):
+        """ Reverse csv field from keys """
+        return [self._config["csv_fields"][k] for k in keys]
+
     def get_tribes(self, only_names=False):
-        """ Get tribes list """
+        """Get tribes list
+
+        :example:
+        >>> loader = CSVLoader("./test_config.yml")
+        >>> loader.get_tribes()
+        {'Tribe1': {'name': 'Tribe1', 'type': 'Type1', 'students': 'tribe1.csv'}, 'Tribe2': {'name': 'Tribe2', 'students': 'tribe2.csv'}}
+        >>> loader.get_tribes(only_names=True)
+        ['Tribe1', 'Tribe2']
+        """
         if only_names:
             return list(self._config["tribes"].keys())
         return self._config["tribes"]
@@ -81,46 +62,52 @@ class CSVLoader(Loader):
         """Get exams list
 
         :param tribes: get only exams for those tribes
-        :return: list of dictionaries of exams (fields: `["name", "tribe", "date", "term"])
+        :return: list of dictionaries of exams (fields: `["exam", "tribe", "date", "term", "score_file"]`)
+
+        :example:
+        >>> loader = CSVLoader("./test_config.yml")
+        >>> loader.get_exams(["Tribe1"])
+           Nom        Date  Trimestre                     score_file   tribe
+        0   DS  12/01/2021          1   example/Tribe1/210112_DS.csv  Tribe1
+        0  DS6  22/01/2021          1  example/Tribe1/210122_DS6.csv  Tribe1
         """
         exams = []
         for tribe in tribes:
-            csvs = list_csvs()
+            tribe_path = Path(self._config["source"]) / tribe
+            csvs = list_csvs(tribe_path)
             for csv in csvs:
-                fields = [
+                fields = self.reverse_csv_field(["exam", "date", "term"])
-                    self._config["csv_fields"][k] for k in ["exam", "date", "term"]
-                ]
                 exam = extract_fields(csv, fields)
-                exam.rename(columns=self._config["csv_fields"], inplace=True).rename(
+                exam = self.rename_columns(exam)
-                    columns={"exam": "name"}, inplace=True
+                exam["score_file"] = csv
-                )
+                exam["tribe"] = tribe
-
+                exams.append(exam)
-        return df.concate(exams)
+        return pd.concat(exams)
 
     def get_students(self, tribes=[]):
         """Get student list
 
         :param filters: list of filters
         """
-        pass
+        return ""
 
     def get_exam_questions(self, exams=[]):
         """Get questions for the exam
 
         :param exams: questions for those exams only
         """
-        pass
+        return ""
 
     def get_questions_scores(self, questions=[]):
         """Get scores of those questions
 
         :param questions: score for those questions
         """
-        pass
+        return ""
 
     def get_student_scores(self, student):
         """Get scores of the student
 
         :param student:
         """
-        pass
+        return ""

test_config.yml

@@ -0,0 +1,13 @@
+---
+source: ./example
+output: ./output
+templates: templates/
+
+tribes:
+    Tribe1:
+        name: Tribe1
+        type: Type1
+        students: tribe1.csv
+    Tribe2:
+        name: Tribe2
+        students: tribe2.csv