Feat: can read exam config from yaml

.gitignore

@@ -122,3 +122,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# vim
+.vim
+

README.md

@@ -6,3 +6,29 @@ Cette fois ci, on utilise:
 - Des fichiers yaml pour les infos sur les élèves
 - Des notebooks pour l'analyse
 - Papermill pour produire les notesbooks à partir de template
+
+## Les fichiers CSV
+
+les paramètres sont décris dans ./recopytex/config.py
+
+### Descriptions des questions
+
+- Trimestre
+- Nom
+- Date
+- Exercice
+- Question
+- Competence
+- Domaine
+- Commentaire
+- Bareme
+- Est_nivele
+
+
+### Valeurs pour notes les élèves
+
+- Score: 0, 1, 2, 3
+- Pas de réponses: .
+- Absent: a
+- Dispensé: (vide)
+

example/recoconfig.yml

@@ -0,0 +1,32 @@
+---
+source: ./
+output: ./
+templates: templates/
+
+competences:
+  Calculer:
+      name: Calculer
+      abrv: Cal
+  Représenter:
+      name: Représenter
+      abrv: Rep
+  Modéliser:
+      name: Modéliser
+      abrv: Mod
+  Raisonner:
+    name: Raisonner
+    abrv: Rai
+  Calculer:
+    name: Calculer
+    abrv: Cal
+  Communiquer:
+    name: Communiquer
+    abrv: Com
+
+
+tribes:
+  - name: Tribe1
+    type: Type1
+    students: tribe1.csv
+  - name: Tribe2
+    students: tribe2.csv

example/tribe1.csv

@@ -0,0 +1,21 @@
+Nom,email
+Star Tice,stice0@jalbum.net
+Umberto Dingate,udingate1@tumblr.com
+Starlin Crangle,scrangle2@wufoo.com
+Humbert Bourcq,hbourcq3@g.co
+Gabriella Handyside,ghandyside4@patch.com
+Stewart Eaves,seaves5@ycombinator.com
+Erick Going,egoing6@va.gov
+Ase Praton,apraton7@va.gov
+Rollins Planks,rplanks8@delicious.com
+Dunstan Sarjant,dsarjant9@naver.com
+Stacy Guiton,sguitona@themeforest.net
+Ange Stanes,astanesb@marriott.com
+Amabelle Elleton,aelletonc@squidoo.com
+Darn Broomhall,dbroomhalld@cisco.com
+Dyan Chatto,dchattoe@npr.org
+Keane Rennebach,krennebachf@dot.gov
+Nari Paulton,npaultong@gov.uk
+Brandy Wase,bwaseh@ftc.gov
+Jaclyn Firidolfi,jfiridolfii@reuters.com
+Violette Lockney,vlockneyj@chron.com

example/tribe2.csv

@@ -0,0 +1,21 @@
+Nom,email
+Elle McKintosh,emckintosh0@1und1.de
+Ty Megany,tmegany1@reuters.com
+Pippa Borrows,pborrows2@a8.net
+Sonny Eskrick,seskrick3@123-reg.co.uk
+Mollee Britch,mbritch4@usda.gov
+Ingram Plaistowe,iplaistowe5@purevolume.com
+Fay Vanyard,fvanyard6@sbwire.com
+Nancy Rase,nrase7@omniture.com
+Rachael Ruxton,rruxton8@bravesites.com
+Tallie Rushmer,trushmer9@home.pl
+Seward MacIlhagga,smacilhaggaa@hatena.ne.jp
+Lizette Searl,lsearlb@list-manage.com
+Talya Mannagh,tmannaghc@webnode.com
+Jordan Witherbed,jwitherbedd@unesco.org
+Reagan Botcherby,rbotcherbye@scientificamerican.com
+Libbie Shoulder,lshoulderf@desdev.cn
+Abner Khomich,akhomichg@youtube.com
+Zollie Kitman,zkitmanh@forbes.com
+Fiorenze Durden,fdurdeni@feedburner.com
+Kevyn Race,kracej@seattletimes.com

recopytex/csv_extraction.py

@@ -17,7 +17,7 @@ def try_replace(x, old, new):
 
 
 def extract_students(df, no_student_columns=NO_ST_COLUMNS.values()):
-    """ Extract the list of students from df 
+    """Extract the list of students from df
 
     :param df: the dataframe
     :param no_student_columns: columns that are not students
@@ -30,7 +30,7 @@ def extract_students(df, no_student_columns=NO_ST_COLUMNS.values()):
 def flat_df_students(
     df, no_student_columns=NO_ST_COLUMNS.values(), postprocessing=True
 ):
-    """ Flat the dataframe by returning a dataframe with on student on each line
+    """Flat the dataframe by returning a dataframe with on student on each line
 
     :param df: the dataframe (one row per questions)
     :param no_student_columns: columns that are not students
@@ -63,7 +63,7 @@ def flat_df_students(
 def flat_df_for(
     df, student, no_student_columns=NO_ST_COLUMNS.values(), postprocessing=True
 ):
-    """ Extract the data only for one student
+    """Extract the data only for one student
 
     :param df: the dataframe (one row per questions)
     :param no_student_columns: columns that are not students
@@ -88,7 +88,7 @@ def flat_df_for(
 
 
 def postprocess(df):
-    """ Postprocessing score dataframe 
+    """Postprocessing score dataframe
 
     - Replace na with an empty string
     - Replace "NOANSWER" with -1

recopytex/df_marks_manip.py

@@ -6,7 +6,9 @@ import numpy as np
 from math import ceil, floor
 from .config import COLUMNS, VALIDSCORE
 
-# Values manipulations
+"""
+Functions for manipulate score dataframes
+"""
 
 
 def round_half_point(val):
@@ -19,12 +21,13 @@ def round_half_point(val):
 
 
 def score_to_mark(x):
-    """ Compute the mark
+    """Compute the mark
 
     if the item is leveled then the score is multiply by the score_rate
     otherwise it copies the score
 
     :param x: dictionnary with COLUMNS["is_leveled"], COLUMNS["score"] and COLUMNS["score_rate"] keys
+    :return: the mark
 
     >>> d = {"Eleve":["E1"]*6 + ["E2"]*6,
     ...    COLUMNS["score_rate"]:[1]*2+[2]*2+[2]*2 + [1]*2+[2]*2+[2]*2,
@@ -43,7 +46,9 @@ def score_to_mark(x):
 
     if x[COLUMNS["is_leveled"]]:
         if x[COLUMNS["score"]] not in [0, 1, 2, 3]:
-            raise ValueError(f"The evaluation is out of range: {x[COLUMNS['score']]} at {x}")
+            raise ValueError(
+                f"The evaluation is out of range: {x[COLUMNS['score']]} at {x}"
+            )
         return round_half_point(x[COLUMNS["score"]] * x[COLUMNS["score_rate"]] / 3)
 
     if x[COLUMNS["score"]] > x[COLUMNS["score_rate"]]:
@@ -54,9 +59,10 @@ def score_to_mark(x):
 
 
 def score_to_level(x):
-    """ Compute the level (".",0,1,2,3).
+    """Compute the level (".",0,1,2,3).
 
     :param x: dictionnary with COLUMNS["is_leveled"], COLUMNS["score"] and COLUMNS["score_rate"] keys
+    :return: the level
 
     >>> d = {"Eleve":["E1"]*6 + ["E2"]*6,
     ...    COLUMNS["score_rate"]:[1]*2+[2]*2+[2]*2 + [1]*2+[2]*2+[2]*2,
@@ -91,7 +97,9 @@ def score_to_level(x):
 
 
 def compute_mark(df):
-    """ Add Mark column to df
+    """Compute the mark for the dataframe
+
+    apply score_to_mark to each row
 
     :param df: DataFrame with COLUMNS["score"], COLUMNS["is_leveled"] and COLUMNS["score_rate"] columns.
 
@@ -122,9 +130,12 @@ def compute_mark(df):
 
 
 def compute_level(df):
-    """ Add Mark column to df
+    """Compute level for the dataframe
+
+    Applies score_to_level to each row
 
     :param df: DataFrame with COLUMNS["score"], COLUMNS["is_leveled"] and COLUMNS["score_rate"] columns.
+    :return: Columns with level
 
     >>> d = {"Eleve":["E1"]*6 + ["E2"]*6,
     ...    COLUMNS["score_rate"]:[1]*2+[2]*2+[2]*2 + [1]*2+[2]*2+[2]*2,
@@ -153,9 +164,10 @@ def compute_level(df):
 
 
 def compute_normalized(df):
-    """ Compute the normalized mark (Mark / score_rate)
+    """Compute the normalized mark (Mark / score_rate)
 
     :param df: DataFrame with "Mark" and COLUMNS["score_rate"] columns
+    :return: column with normalized mark
 
     >>> d = {"Eleve":["E1"]*6 + ["E2"]*6,
     ...    COLUMNS["score_rate"]:[1]*2+[2]*2+[2]*2 + [1]*2+[2]*2+[2]*2,
@@ -186,7 +198,9 @@ def compute_normalized(df):
 
 
 def pp_q_scores(df):
-    """ Postprocessing questions scores dataframe
+    """Postprocessing questions scores dataframe
+
+    Add 3 columns: mark, level and normalized
 
     :param df: questions-scores dataframe
     :return: same data frame with mark, level and normalize columns

recopytex/scripts/getconfig.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# encoding: utf-8
+import yaml
+
+CONFIGPATH = "recoconfig.yml"
+
+with open(CONFIGPATH, "r") as config:
+    config = yaml.load(config, Loader=yaml.FullLoader)
+

recopytex/scripts/prompts.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+
+from prompt_toolkit import prompt, HTML, ANSI
+from prompt_toolkit import print_formatted_text as print
+from prompt_toolkit.styles import Style
+from prompt_toolkit.validation import Validator
+from prompt_toolkit.completion import WordCompleter
+from unidecode import unidecode
+from datetime import datetime
+from functools import wraps
+import sys
+
+from .getconfig import config
+
+
+VALIDATE = [
+    "o",
+    "ok",
+    "OK",
+    "oui",
+    "OUI",
+    "yes",
+    "YES",
+]
+REFUSE = ["n", "non", "NON", "no", "NO"]
+CANCEL = ["a", "annuler"]
+
+STYLE = Style.from_dict(
+    {
+        "": "#93A1A1",
+        "validation": "#884444",
+        "appending": "#448844",
+    }
+)
+
+
+class CancelError(Exception):
+    pass
+
+
+def prompt_validate(question, cancelable=False, empty_means=1, style="validation"):
+    """Prompt for validation
+
+    :param question: Text to print to ask the question.
+    :param cancelable: enable cancel answer
+    :param empty_means: result for no answer
+    :return:
+        0 -> Refuse
+        1 -> Validate
+        -1 -> cancel
+    """
+    question_ = question
+    choices = VALIDATE + REFUSE
+
+    if cancelable:
+        question_ += "(a ou annuler pour sortir)"
+        choices += CANCEL
+
+    ans = prompt(
+        [
+            (f"class:{style}", question_),
+        ],
+        completer=WordCompleter(choices),
+        style=STYLE,
+    ).lower()
+
+    if ans == "":
+        return empty_means
+    if ans in VALIDATE:
+        return 1
+    if cancelable and ans in CANCEL:
+        return -1
+    return 0
+
+
+def prompt_until_validate(question="C'est ok? ", cancelable=False):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwrd):
+            ans = func(*args, **kwrd)
+
+            confirm = prompt_validate(question, cancelable)
+
+            if confirm == -1:
+                raise CancelError
+
+            while not confirm:
+                sys.stdout.flush()
+                ans = func(*args, **ans, **kwrd)
+                confirm = prompt_validate(question, cancelable)
+                if confirm == -1:
+                    raise CancelError
+            return ans
+
+        return wrapper
+
+    return decorator
+
+
+@prompt_until_validate()
+def prompt_exam(**kwrd):
+    """ Prompt questions to edit an exam """
+    print(HTML("<b>Nouvelle évaluation</b>"))
+    exam = {}
+    exam["name"] = prompt("Nom de l'évaluation: ", default=kwrd.get("name", "DS"))
+
+    tribes_name = [t["name"] for t in config["tribes"]]
+
+    exam["tribename"] = prompt(
+        "Nom de la classe: ",
+        default=kwrd.get("tribename", ""),
+        completer=WordCompleter(tribes_name),
+        validator=Validator.from_callable(lambda x: x in tribes_name),
+    )
+    exam["tribe"] = [t for t in config["tribes"] if t["name"] == exam["tribename"]][0]
+
+    exam["date"] = prompt(
+        "Date de l'évaluation (%y%m%d): ",
+        default=kwrd.get("date", datetime.today()).strftime("%y%m%d"),
+        validator=Validator.from_callable(lambda x: (len(x) == 6) and x.isdigit()),
+    )
+    exam["date"] = datetime.strptime(exam["date"], "%y%m%d")
+
+    exam["term"] = prompt(
+        "Trimestre: ",
+        validator=Validator.from_callable(lambda x: x.isdigit()),
+        default=kwrd.get("term", "1"),
+    )
+
+    return exam
+
+
+@prompt_until_validate()
+def prompt_exercise(number=1, **kwrd):
+    try:
+        kwrd["name"]
+    except KeyError:
+        print(HTML("<b>Nouvel exercice</b>"))
+    else:
+        print(HTML(f"<b>Modification de l'exercice: {kwrd['name']}</b>"))
+
+    exercise = {}
+    exercise["name"] = prompt(
+        "Nom de l'exercice: ", default=kwrd.get("name", f"Exercice {number}")
+    )
+
+    exercise["questions"] = []
+
+    try:
+        kwrd["questions"][0]
+    except KeyError:
+        last_question_id = "1a"
+    except IndexError:
+        last_question_id = "1a"
+    else:
+        for ques in kwrd["questions"]:
+            try:
+                exercise["questions"].append(prompt_question(**ques))
+            except CancelError:
+                print("Cette question a été supprimée")
+        last_question_id = exercise["questions"][-1]["id"]
+
+    appending = prompt_validate(
+        question="Ajouter un élément de notation? ", style="appending"
+    )
+    while appending:
+        try:
+            exercise["questions"].append(prompt_question(last_question_id))
+        except CancelError:
+            print("Cette question a été supprimée")
+        else:
+            last_question_id = exercise["questions"][-1]["id"]
+        appending = prompt_validate(
+            question="Ajouter un élément de notation? ", style="appending"
+        )
+
+    return exercise
+
+
+@prompt_until_validate(cancelable=True)
+def prompt_question(last_question_id="1a", **kwrd):
+    try:
+        kwrd["id"]
+    except KeyError:
+        print(HTML("<b>Nouvel élément de notation</b>"))
+    else:
+        print(
+            HTML(
+                f"<b>Modification de l'élément {kwrd['id']} ({kwrd['comment']})</b>"
+            )
+        )
+
+    question = {}
+    question["id"] = prompt(
+        "Identifiant de la question: ",
+        default=kwrd.get("id", "1a"),
+    )
+
+    question["competence"] = prompt(
+        "Competence: ",
+        default=kwrd.get("competence", list(config["competences"].keys())[0]),
+        completer=WordCompleter(config["competences"].keys()),
+        validator=Validator.from_callable(lambda x: x in config["competences"].keys()),
+    )
+
+    question["theme"] = prompt(
+        "Domaine: ",
+        default=kwrd.get("theme", ""),
+        # completer
+    )
+
+    question["comment"] = prompt(
+        "Commentaire: ",
+        default=kwrd.get("comment", ""),
+    )
+
+    question["is_leveled"] = prompt(
+        "Évaluation par niveau: ",
+        default=kwrd.get("is_leveled", "1"),
+        # validate
+    )
+
+    question["score_rate"] = prompt(
+        "Barème: ",
+        default=kwrd.get("score_rate", "1"),
+        # validate
+    )
+
+    return question

recopytex/scripts/recopytex.py

@@ -3,15 +3,15 @@
 
 import click
 from pathlib import Path
-import yaml
 import sys
 import papermill as pm
+import pandas as pd
 from datetime import datetime
+import yaml
 
-CONFIGPATH = "recoconfig.yml"
-
-with open(CONFIGPATH, "r") as config:
-    config = yaml.load(config, Loader=yaml.FullLoader)
+from .getconfig import config, CONFIGPATH
+from .prompts import prompt_exam, prompt_exercise, prompt_validate
+from ..config import NO_ST_COLUMNS
 
 
 @click.group()
@@ -26,6 +26,81 @@ def print_config():
     click.echo(config)
 
 
+@cli.command()
+def setup():
+    """Setup the environnement using recoconfig.yml"""
+    for tribe in config["tribes"]:
+        Path(tribe["name"]).mkdir(exist_ok=True)
+        if not Path(tribe["students"]).exists():
+            print(f"The file {tribe['students']} does not exists")
+
+
+def exam_dict2row(exam):
+    """ Transform an exam in dictionnary for into list of rows to evaluate"""
+    rows = []
+    for ex in exam["exercices"]:
+        for q in ex["questions"]:
+            rows.append(
+                {
+                    "term": exam["term"],
+                    "assessment": exam["name"],
+                    "date": exam["date"].strftime("%d/%m/%Y"),
+                    "exercise": ex["name"],
+                    "question": q["id"],
+                    **q,
+                }
+            )
+    return rows
+
+def get_exam_name(exam):
+    """ Get exam name from exam data """
+    return f"{exam['date'].strftime('%y%m%d')}_{exam['name']}"
+
+def get_tribe_path(exam):
+    """ Get tribe  path from exam data """
+    return Path(config["source"]) / exam["tribe"]["name"]
+
+def get_exam_path(exam, extention=""):
+    return get_tribe_path(exam)/ (get_exam_name(exam) + extention)
+
+
+@cli.command()
+def new_exam():
+    """ Create new exam csv file """
+    exam = prompt_exam()
+
+    if get_exam_path(exam, ".yml").exists():
+        with open(get_exam_path(exam, ".yml"), "r") as f:
+            exam["exercices"] = yaml.load(f, Loader=yaml.SafeLoader)["exercices"]
+    else:
+        exam["exercices"] = []
+
+    for i, ex in enumerate(exam["exercices"]):
+        exam["exercices"][i] = prompt_exercise(**ex)
+
+    new_exercise = prompt_validate("Ajouter un exercice? ")
+    while new_exercise:
+        exam["exercices"].append(prompt_exercise(len(exam["exercices"])+1))
+        new_exercise = prompt_validate("Ajouter un exercice? ")
+
+
+    rows = exam_dict2row(exam)
+
+    base_df = pd.DataFrame.from_dict(rows)[NO_ST_COLUMNS.keys()]
+    base_df.rename(columns=NO_ST_COLUMNS, inplace=True)
+
+    students = pd.read_csv(exam["tribe"]["students"])["Nom"]
+    for student in students:
+        base_df[student] = ""
+
+    path = Path(config["source"]) / exam["tribe"]["name"]
+    path.mkdir(exist_ok=True)
+
+    dest = path / get_exam_name(exam) + ".csv"
+    base_df.to_csv(dest, index=False)
+    print(f"Le fichier note a été enregistré à {dest}")
+
+
 @cli.command()
 @click.argument("csv_file")
 def report(csv_file):
@@ -60,22 +135,9 @@ def report(csv_file):
         str(template),
         str(dest / f"{assessment}.ipynb"),
         parameters=dict(
-            tribe=tribe, assessment=assessment, date=f"{date:%d/%m/%y}", csv_file=str(csv_file.absolute())
+            tribe=tribe,
+            assessment=assessment,
+            date=f"{date:%d/%m/%y}",
+            csv_file=str(csv_file.absolute()),
         ),
     )
-
-    # with open(csv_file.parent / "description.yml") as f:
-    #     tribe_desc = yaml.load(f, Loader=yaml.FullLoader)
-
-    # template = Path(config["templates"]) / "tpl_student.ipynb"
-    # dest = Path(config["output"]) / tribe / csv_filename / "students"
-    # dest.mkdir(parents=True, exist_ok=True)
-
-    # for st in tribe_desc["students"]:
-    #     click.echo(f"Building {st} report on {assessment}")
-    #     pm.execute_notebook(
-    #         str(template),
-    #         str(dest / f"{st}.ipynb"),
-    #         parameters=dict(tribe=tribe, student=st, source=str(tribe_dir.absolute())),
-    #     )
-

requirements.txt

@@ -1,69 +1,4 @@
-ansiwrap==0.8.4
-attrs==19.1.0
-backcall==0.1.0
-bleach==3.1.0
-certifi==2019.6.16
-chardet==3.0.4
-Click==7.0
-colorama==0.4.1
-cycler==0.10.0
-decorator==4.4.0
-defusedxml==0.6.0
-entrypoints==0.3
-future==0.17.1
-idna==2.8
-importlib-resources==1.0.2
-ipykernel==5.1.1
-ipython==7.7.0
-ipython-genutils==0.2.0
-ipywidgets==7.5.1
-jedi==0.14.1
-Jinja2==2.10.1
-jsonschema==3.0.2
-jupyter==1.0.0
-jupyter-client==5.3.1
-jupyter-console==6.0.0
-jupyter-core==4.5.0
-jupytex==0.0.3
-kiwisolver==1.1.0
-MarkupSafe==1.1.1
-matplotlib==3.1.1
-mistune==0.8.4
-nbconvert==5.5.0
-nbformat==4.4.0
-notebook==6.0.0
-numpy==1.17.0
-pandas==0.25.0
-pandocfilters==1.4.2
-papermill==1.0.1
-parso==0.5.1
-pexpect==4.7.0
-pickleshare==0.7.5
-prometheus-client==0.7.1
-prompt-toolkit==2.0.9
-ptyprocess==0.6.0
-Pygments==2.4.2
-pyparsing==2.4.2
-pyrsistent==0.15.4
-python-dateutil==2.8.0
-pytz==2019.2
-PyYAML==5.1.2
-pyzmq==18.0.2
-qtconsole==4.5.2
--e git+git_opytex:/lafrite/recopytex.git@e9a8310f151ead60434ae944d726a2fd22b23d06#egg=Recopytex
-requests==2.22.0
-scipy==1.3.0
-seaborn==0.9.0
-Send2Trash==1.5.0
-six==1.12.0
-tenacity==5.0.4
-terminado==0.8.2
-testpath==0.4.2
-textwrap3==0.9.2
-tornado==6.0.3
-tqdm==4.32.2
-traitlets==4.3.2
-urllib3==1.25.3
-wcwidth==0.1.7
-webencodings==0.5.1
-widgetsnbextension==3.5.1
+pandas
+click
+papermill
+prompt_toolkit

requirements_dev.txt

@@ -0,0 +1,69 @@
+ansiwrap
+attrs
+backcall
+bleach
+certifi
+chardet
+Click
+colorama
+cycler
+decorator
+defusedxml
+entrypoints
+future
+idna
+importlib-resources
+ipykernel
+ipython
+ipython-genutils
+ipywidgets
+jedi
+Jinja2
+jsonschema
+jupyter
+jupyter-client
+jupyter-console
+jupyter-core
+jupytex
+kiwisolver
+MarkupSafe
+matplotlib
+mistune
+nbconvert
+nbformat
+notebook
+numpy
+pandas
+pandocfilters
+papermill
+parso
+pexpect
+pickleshare
+prometheus-client
+prompt-toolkit
+ptyprocess
+Pygments
+pyparsing
+pyrsistent
+python-dateutil
+pytz
+PyYAML
+pyzmq
+qtconsole
+-e git+git_opytex:/lafrite/recopytex.git@e9a8310f151ead60434ae944d726a2fd22b23d06#egg=Recopytex
+requests
+scipy
+seaborn
+Send2Trash
+six
+tenacity
+terminado
+testpath
+textwrap3
+tornado
+tqdm
+traitlets
+urllib3
+wcwidth
+webencodings
+widgetsnbextension