Feat: callback to toggle editing

dashboard/app.py

@@ -0,0 +1,60 @@
+import dash
+from dash import Dash, html, dcc
+from .pages import home, config, stage, schema, table
+
+external_scripts = [
+    {'src': 'https://cdn.tailwindcss.com'}
+]
+# external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}]
+
+app = Dash(__name__,
+           use_pages=True,
+           external_scripts=external_scripts,
+           suppress_callback_exceptions=True,
+           )
+app.scripts.config.serve_locally = True
+dash.register_page(
+    home.__name__, 
+    path='/', 
+    layout=home.layout,
+)
+dash.register_page(
+    config.__name__, 
+    path='/config', 
+    layout=config.layout
+)
+dash.register_page(
+    stage.__name__, 
+    path_template='/stage/<stage_name>', 
+    layout=stage.layout
+)
+dash.register_page(
+    schema.__name__, 
+    path_template='/stg/<stage_name>/schema/<schema_name>',
+    layout=schema.layout
+)
+dash.register_page(
+    table.__name__,
+    path_template='/stg/<stage_name>/schm/<schema_name>/table/<table_name>',
+    layout=table.layout
+)
+
+app.layout = html.Div([
+    html.Div([
+        dcc.Link(
+            html.H1('Plesna', ),
+            href="/",
+            className="text-4xl p-4 text-center grow align-baseline"
+        ),
+        dcc.Link("Config",
+                 href="/config",
+                 className="flex-none hover:bg-amber-100 p-4 align-middle"
+                 )
+    ],
+        className="bg-amber-300 flex flex-row shadow"
+    ),
+    dash.page_container
+])
+
+if __name__ == '__main__':
+    app.run(debug=True)

dashboard/components/lists.py

@@ -0,0 +1,55 @@
+from ..libs.stage.stage import AbstractStage
+from dash import html, dcc
+
+def html_list_schema(stage:AbstractStage, with_tables=True):
+    """ Build html list of schema in stage """
+    ul_classes = "ml-2"
+    schema_baseurl = f"/stg/{stage.name}/schema/"
+    if with_tables:
+        return html.Ul(
+            [
+                html.Li(
+                    children = [
+                        dcc.Link(
+                            schema,
+                            href=schema_baseurl + schema,
+                            className="text-lg hover:underline"
+                        ),
+                        html_list_table(stage, schema)
+                    ],
+                    className=""
+                ) for schema in stage.schemas()
+            ],
+            className=ul_classes
+        )
+    return html.Ul(
+        [
+            html.Li(
+                dcc.Link(
+                    schema,
+                    href=schema_baseurl + schema,
+                    className="text-lg hover:underline"
+                ),
+            ) for schema in stage.schemas()
+        ],
+        className=ul_classes
+    )
+
+
+def html_list_table(stage:AbstractStage, schema:str):
+    """ Build html list of table in stage """
+    table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/"
+    return html.Ul(
+        [
+            html.Li(
+                dcc.Link(
+                    table,
+                    href=table_baseurl + table,
+                    className="hover:underline"
+                ),
+            ) for table in stage.tables(schema=schema)
+        ],
+        className="ml-4"
+    )
+
+

dashboard/datalake.py

@@ -0,0 +1,13 @@
+from .libs.stage.fs_stage import FSStage
+from dotenv import dotenv_values
+
+env = {
+    **dotenv_values(".env"),
+}
+
+stages = {
+    "raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
+    "staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
+    "gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
+    "mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
+}

dashboard/libs/fs_schema.py

@@ -0,0 +1,35 @@
+from .schema import AbstractSchema
+from pathlib import Path
+
+class FSSchema(AbstractSchema):
+    def __init__(self, basepath, metadata_engine=None):
+        self.basepath = basepath
+        self._metadata_engine = metadata_engine
+
+    def ls(self, dir, only_files=True):
+        dirpath = Path(dir)
+        if only_files:
+            return [f for f in dirpath.iterdir() if f.is_dir()]
+        return [f for f in dirpath.iterdir()]
+
+    def tables(self, dir, only_files=True):
+        dirpath = Path(dir)
+        if only_files:
+            return [f for f in dirpath.iterdir() if f.is_dir()]
+        return [f for f in dirpath.iterdir()]
+
+    def info(self, path):
+        path = Path(path)
+        pass
+
+    def read(self, path):
+        path = Path(path)
+        pass
+
+    def write(self, path, content):
+        path = Path(path)
+        pass
+
+    def delete(self, path):
+        path = Path(path)
+        pass

dashboard/libs/stage/fs_stage.py

@@ -0,0 +1,61 @@
+from .stage import AbstractStage
+from pathlib import Path
+import pandas as pd
+
+class FSStage(AbstractStage):
+    def __init__(self, name, basepath, metadata_engine=None):
+        self.name = name
+
+        self.basepath = Path(basepath)
+        self._metadata_engine = metadata_engine
+
+    def ls(self, dir, only_files=False, only_directories=False, recursive=False) -> list[str]:
+        dirpath = Path(dir)
+
+        if only_files:
+            return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if not f.is_dir()]
+
+        if only_directories:
+            if recursive:
+                return [str(f[0].relative_to(dirpath)) for f in dirpath.walk()]
+
+            return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if f.is_dir()]
+
+        return [str(f.relative_to(dirpath)) for f in dirpath.iterdir()]
+
+    def schemas(self, recursive=True) -> list[str]:
+        dirpath = self.basepath
+        return self.ls(dirpath, only_directories=True, recursive=True)
+
+    def tables(self, schema:str) -> list[str]:
+        dirpath = self.basepath / schema
+        return self.ls(dirpath, only_files=True)
+
+    def build_table_path(self, table:str, schema:str):
+        table_path = self.basepath
+        if schema == '.':
+            return table_path / table
+        return table_path / schema / table
+
+    def info(self, table:str, schema:str='.'):
+        table_path = self.build_table_path(table, schema)
+        pass
+
+    def read(self, table:str, schema:str='.', read_options={}):
+        table_path = self.build_table_path(table, schema)
+        extension = table_path.suffix
+        if extension == '.csv':
+            return pd.read_csv(table_path, **read_options)
+
+        if extension == '.xlsx':
+            return pd.read_excel(table_path, **read_options)
+
+        raise ValueError("Can't open the table")
+
+    def write(self, table:str, content, schema:str='.'):
+        table_path = self.build_table_path(table, schema)
+        pass
+
+    def delete(self, table:str, schema:str='.'):
+        table_path = self.build_table_path(table, schema)
+        pass

dashboard/libs/stage/metadata.py

@@ -0,0 +1,5 @@
+from abc import ABC
+
+
+class AbstractMetadataEngine(ABC):
+    pass

dashboard/libs/stage/stage.py

@@ -0,0 +1,36 @@
+import abc
+from .metadata import AbstractMetadataEngine
+
+
+class AbstractStage(abc.ABC):
+    metadata_engine = AbstractMetadataEngine
+
+    @abc.abstractmethod
+    def schemas():
+        """ List schemas """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def tables(schema):
+        """ List table in schema"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def info(self, path):
+        """ Get infos about a file"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def read(self, path):
+        """ Get content of a file"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def write(self, path, content):
+        """ Write content into the file"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, path):
+        """ Delete the file """
+        raise NotImplementedError

dashboard/pages/config.py

@@ -0,0 +1,14 @@
+from dash import html
+from dotenv import dotenv_values
+import os
+
+env = {
+    **dotenv_values(".env"),
+    **os.environ,
+}
+
+
+layout = html.Div([
+    html.H1('This is our Config page'),
+    html.Ul(children = [html.Li(f"{k} = {v}") for k,v in env.items()]),
+])

dashboard/pages/home.py

@@ -0,0 +1,27 @@
+from dash import html, dcc
+from ..datalake import stages
+from ..components.lists import html_list_schema
+
+
+layout = html.Div([
+    html.Div(children=[
+        html.Ul(
+            children=[
+                html.Li(
+                    children=[
+                        dcc.Link(
+                            stagename,
+                            href=f"/stage/{stagename}",
+                            className="text-2xl text-center p-2 bg-amber-100 rounded shadow"
+                        ),
+                        html_list_schema(stage)
+                    ],
+                    className="flex-1 bg-gray-100 rounded flex flex-col shadow"
+                ) for stagename, stage in stages.items()
+            ],
+            className="flex flex-row space-x-2"
+        )
+    ],
+    className="w-full mt-4 px-2"
+    ),
+])

dashboard/pages/schema.py

@@ -0,0 +1,24 @@
+from dash import html, dcc
+from ..datalake import stages
+from ..libs.stage.stage import AbstractStage
+
+
+def layout(stage_name=None, schema_name=None):
+    stage = stages[stage_name]
+    return html.Div([
+        html.H2([
+            dcc.Link(
+                f"{stage.name}",
+                href=f"/stage/{stage.name}",
+                className="hover:underline"
+            ),
+            html.Span(" > "),
+            html.Span(
+                f"{schema_name}",
+            ),
+        ],
+            className="text-2xl p-4 py-2"
+
+        ),
+    ])
+

dashboard/pages/stage.py

@@ -0,0 +1,18 @@
+from dash import html, dcc
+from ..datalake import stages
+from ..libs.stage.stage import AbstractStage
+from ..components.lists import html_list_schema
+
+
+def layout(stage_name=None):
+    stage = stages[stage_name]
+    return html.Div([
+
+        html.H2(
+            f"{stage.name}",
+            className="text-2xl p-4 py-2"
+        ),
+            html_list_schema(stage)
+        ],
+        className = "flex flex-col"
+    )

dashboard/pages/table.py

@@ -0,0 +1,126 @@
+from dash import html, dcc, dash_table, callback, Input, Output, State
+from dash.exceptions import PreventUpdate
+from ..datalake import stages
+from ..libs.stage.stage import AbstractStage
+
+
+def layout(stage_name=None, schema_name=None, table_name=None):
+    stage = stages[stage_name]
+    df = stage.read(table=table_name, schema=schema_name)
+    return html.Div([
+        dcc.Store(id="table_backup"),
+        html.Div([
+            html.H2([
+                dcc.Link(
+                    f"{stage.name}",
+                    href=f"/stage/{stage.name}",
+                    className="hover:underline"
+                ),
+                html.Span(" > "),
+                dcc.Link(
+                    f"{schema_name}",
+                    href=f"/stg/{stage.name}/schema/{schema_name}",
+                    className="hover:underline"
+                ),
+                html.Span(" > "),
+                html.Span(table_name),
+            ],
+            className="text-2xl"
+            ),
+            html.Div([
+                html.Button(
+                    "Editer",
+                    id="btn_edit",
+                    className="rounded border px-2 py-1",
+                    style={"display": "block"}
+                ),
+                html.Button(
+                    "Sauver",
+                    id="btn_save",
+                    className="rounded border px-2 py-1 border-green-500 hover:bg-green-500",
+                    style={"display": "none"}
+                ),
+                html.Button(
+                    "Annuler",
+                    id="btn_cancel",
+                    className="rounded border px-2 py-1 border-red-500 hover:bg-red-500",
+                    style={"display": "none"}
+                ),
+            ],
+            className="flex flex-row space-x-2",
+            id="toolbar"
+            ),
+        ],
+        className="flex flex-row justify-between p-4"
+        ),
+        html.Div([
+            html.Div([
+                dash_table.DataTable(
+                    id="datatable",
+                    data=df.to_dict('records'),
+                    columns=[{"name": i, "id": i} for i in df.columns],
+                    filter_action="native",
+                    sort_action="native",
+                    sort_mode="multi",
+                    editable=False
+                )
+            ])
+        ],
+     className="overflow-y-auto"
+     ),
+    ],
+    className="p-2"
+    )
+
+
+@callback(
+    Output("datatable", 'editable', allow_duplicate=True),
+    Output("table_backup", 'data'),
+    Input("btn_edit", "n_clicks"),
+    State("datatable", 'data'),
+    prevent_initial_call=True
+)
+def activate_editable(n_clicks, df_src):
+    if n_clicks is None:
+        raise PreventUpdate
+    if n_clicks > 0:
+        df_backup = df_src.copy()
+        return True, df_backup
+    raise PreventUpdate
+
+@callback(
+    Output("datatable", 'editable'),
+    Output("datatable", 'data'),
+    Input("btn_cancel", "n_clicks"),
+    State("table_backup", 'data'),
+    prevent_initial_call=True
+)
+def cancel_modifications(n_clicks, data):
+    if n_clicks is None:
+        raise PreventUpdate
+    if n_clicks > 0 and data is not None:
+        return False, data.copy()
+    raise PreventUpdate
+
+# @callback(
+#     Output("datatable", 'editable'),
+#     Input("btn_save", "n_clicks"),
+#     State("datatable", 'editable'),
+# )
+# def save_modifications(n_clicks, editable):
+#     if n_clicks is None:
+#         raise PreventUpdate
+#     if n_clicks > 0:
+#         return not editable
+#     return editable
+
+@callback(
+    Output("btn_edit", "style"),
+    Output("btn_save", "style"),
+    Output("btn_cancel", "style"),
+    Input("datatable", "editable"),
+)
+def toolbar(editable):
+    if editable:
+        return {"display": "none"}, {"display": "block"}, {"display": "block"}
+    return {"display": "block"}, {"display": "none"}, {"display": "none"}