dashboard/app.py

@@ -1,60 +0,0 @@
-import dash
-from dash import Dash, html, dcc
-from .pages import home, config, stage, schema, table
-
-external_scripts = [
-    {'src': 'https://cdn.tailwindcss.com'}
-]
-# external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}]
-
-app = Dash(__name__,
-           use_pages=True,
-           external_scripts=external_scripts,
-           suppress_callback_exceptions=True,
-           )
-app.scripts.config.serve_locally = True
-dash.register_page(
-    home.__name__, 
-    path='/', 
-    layout=home.layout,
-)
-dash.register_page(
-    config.__name__, 
-    path='/config', 
-    layout=config.layout
-)
-dash.register_page(
-    stage.__name__, 
-    path_template='/stage/<stage_name>', 
-    layout=stage.layout
-)
-dash.register_page(
-    schema.__name__, 
-    path_template='/stg/<stage_name>/schema/<schema_name>',
-    layout=schema.layout
-)
-dash.register_page(
-    table.__name__,
-    path_template='/stg/<stage_name>/schm/<schema_name>/table/<table_name>',
-    layout=table.layout
-)
-
-app.layout = html.Div([
-    html.Div([
-        dcc.Link(
-            html.H1('Plesna', ),
-            href="/",
-            className="text-4xl p-4 text-center grow align-baseline"
-        ),
-        dcc.Link("Config",
-                 href="/config",
-                 className="flex-none hover:bg-amber-100 p-4 align-middle"
-                 )
-    ],
-        className="bg-amber-300 flex flex-row shadow"
-    ),
-    dash.page_container
-])
-
-if __name__ == '__main__':
-    app.run(debug=True)

dashboard/components/lists.py

@@ -1,55 +0,0 @@
-from ..libs.stage.stage import AbstractStage
-from dash import html, dcc
-
-def html_list_schema(stage:AbstractStage, with_tables=True):
-    """ Build html list of schema in stage """
-    ul_classes = "ml-2"
-    schema_baseurl = f"/stg/{stage.name}/schema/"
-    if with_tables:
-        return html.Ul(
-            [
-                html.Li(
-                    children = [
-                        dcc.Link(
-                            schema,
-                            href=schema_baseurl + schema,
-                            className="text-lg hover:underline"
-                        ),
-                        html_list_table(stage, schema)
-                    ],
-                    className=""
-                ) for schema in stage.schemas()
-            ],
-            className=ul_classes
-        )
-    return html.Ul(
-        [
-            html.Li(
-                dcc.Link(
-                    schema,
-                    href=schema_baseurl + schema,
-                    className="text-lg hover:underline"
-                ),
-            ) for schema in stage.schemas()
-        ],
-        className=ul_classes
-    )
-
-
-def html_list_table(stage:AbstractStage, schema:str):
-    """ Build html list of table in stage """
-    table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/"
-    return html.Ul(
-        [
-            html.Li(
-                dcc.Link(
-                    table,
-                    href=table_baseurl + table,
-                    className="hover:underline"
-                ),
-            ) for table in stage.tables(schema=schema)
-        ],
-        className="ml-4"
-    )
-
-

dashboard/datalake.py

@@ -1,13 +0,0 @@
-from .libs.stage.fs_stage import FSStage
-from dotenv import dotenv_values
-
-env = {
-    **dotenv_values(".env"),
-}
-
-stages = {
-    "raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
-    "staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
-    "gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
-    "mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
-}

dashboard/libs/fs_schema.py

@@ -1,35 +0,0 @@
-from .schema import AbstractSchema
-from pathlib import Path
-
-class FSSchema(AbstractSchema):
-    def __init__(self, basepath, metadata_engine=None):
-        self.basepath = basepath
-        self._metadata_engine = metadata_engine
-
-    def ls(self, dir, only_files=True):
-        dirpath = Path(dir)
-        if only_files:
-            return [f for f in dirpath.iterdir() if f.is_dir()]
-        return [f for f in dirpath.iterdir()]
-
-    def tables(self, dir, only_files=True):
-        dirpath = Path(dir)
-        if only_files:
-            return [f for f in dirpath.iterdir() if f.is_dir()]
-        return [f for f in dirpath.iterdir()]
-
-    def info(self, path):
-        path = Path(path)
-        pass
-
-    def read(self, path):
-        path = Path(path)
-        pass
-
-    def write(self, path, content):
-        path = Path(path)
-        pass
-
-    def delete(self, path):
-        path = Path(path)
-        pass

dashboard/libs/stage/fs_stage.py

@@ -1,61 +0,0 @@
-from .stage import AbstractStage
-from pathlib import Path
-import pandas as pd
-
-class FSStage(AbstractStage):
-    def __init__(self, name, basepath, metadata_engine=None):
-        self.name = name
-
-        self.basepath = Path(basepath)
-        self._metadata_engine = metadata_engine
-
-    def ls(self, dir, only_files=False, only_directories=False, recursive=False) -> list[str]:
-        dirpath = Path(dir)
-
-        if only_files:
-            return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if not f.is_dir()]
-
-        if only_directories:
-            if recursive:
-                return [str(f[0].relative_to(dirpath)) for f in dirpath.walk()]
-
-            return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if f.is_dir()]
-
-        return [str(f.relative_to(dirpath)) for f in dirpath.iterdir()]
-
-    def schemas(self, recursive=True) -> list[str]:
-        dirpath = self.basepath
-        return self.ls(dirpath, only_directories=True, recursive=True)
-
-    def tables(self, schema:str) -> list[str]:
-        dirpath = self.basepath / schema
-        return self.ls(dirpath, only_files=True)
-
-    def build_table_path(self, table:str, schema:str):
-        table_path = self.basepath
-        if schema == '.':
-            return table_path / table
-        return table_path / schema / table
-
-    def info(self, table:str, schema:str='.'):
-        table_path = self.build_table_path(table, schema)
-        pass
-
-    def read(self, table:str, schema:str='.', read_options={}):
-        table_path = self.build_table_path(table, schema)
-        extension = table_path.suffix
-        if extension == '.csv':
-            return pd.read_csv(table_path, **read_options)
-
-        if extension == '.xlsx':
-            return pd.read_excel(table_path, **read_options)
-
-        raise ValueError("Can't open the table")
-
-    def write(self, table:str, content, schema:str='.'):
-        table_path = self.build_table_path(table, schema)
-        pass
-
-    def delete(self, table:str, schema:str='.'):
-        table_path = self.build_table_path(table, schema)
-        pass

dashboard/libs/stage/metadata.py

@@ -1,5 +0,0 @@
-from abc import ABC
-
-
-class AbstractMetadataEngine(ABC):
-    pass

dashboard/libs/stage/stage.py

@@ -1,36 +0,0 @@
-import abc
-from .metadata import AbstractMetadataEngine
-
-
-class AbstractStage(abc.ABC):
-    metadata_engine = AbstractMetadataEngine
-
-    @abc.abstractmethod
-    def schemas():
-        """ List schemas """
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def tables(schema):
-        """ List table in schema"""
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def info(self, path):
-        """ Get infos about a file"""
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def read(self, path):
-        """ Get content of a file"""
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def write(self, path, content):
-        """ Write content into the file"""
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def delete(self, path):
-        """ Delete the file """
-        raise NotImplementedError

dashboard/pages/config.py

@@ -1,14 +0,0 @@
-from dash import html
-from dotenv import dotenv_values
-import os
-
-env = {
-    **dotenv_values(".env"),
-    **os.environ,
-}
-
-
-layout = html.Div([
-    html.H1('This is our Config page'),
-    html.Ul(children = [html.Li(f"{k} = {v}") for k,v in env.items()]),
-])

dashboard/pages/home.py

@@ -1,27 +0,0 @@
-from dash import html, dcc
-from ..datalake import stages
-from ..components.lists import html_list_schema
-
-
-layout = html.Div([
-    html.Div(children=[
-        html.Ul(
-            children=[
-                html.Li(
-                    children=[
-                        dcc.Link(
-                            stagename,
-                            href=f"/stage/{stagename}",
-                            className="text-2xl text-center p-2 bg-amber-100 rounded shadow"
-                        ),
-                        html_list_schema(stage)
-                    ],
-                    className="flex-1 bg-gray-100 rounded flex flex-col shadow"
-                ) for stagename, stage in stages.items()
-            ],
-            className="flex flex-row space-x-2"
-        )
-    ],
-    className="w-full mt-4 px-2"
-    ),
-])

dashboard/pages/schema.py

@@ -1,24 +0,0 @@
-from dash import html, dcc
-from ..datalake import stages
-from ..libs.stage.stage import AbstractStage
-
-
-def layout(stage_name=None, schema_name=None):
-    stage = stages[stage_name]
-    return html.Div([
-        html.H2([
-            dcc.Link(
-                f"{stage.name}",
-                href=f"/stage/{stage.name}",
-                className="hover:underline"
-            ),
-            html.Span(" > "),
-            html.Span(
-                f"{schema_name}",
-            ),
-        ],
-            className="text-2xl p-4 py-2"
-
-        ),
-    ])
-

dashboard/pages/stage.py

@@ -1,18 +0,0 @@
-from dash import html, dcc
-from ..datalake import stages
-from ..libs.stage.stage import AbstractStage
-from ..components.lists import html_list_schema
-
-
-def layout(stage_name=None):
-    stage = stages[stage_name]
-    return html.Div([
-
-        html.H2(
-            f"{stage.name}",
-            className="text-2xl p-4 py-2"
-        ),
-            html_list_schema(stage)
-        ],
-        className = "flex flex-col"
-    )

dashboard/pages/table.py

@@ -1,126 +0,0 @@
-from dash import html, dcc, dash_table, callback, Input, Output, State
-from dash.exceptions import PreventUpdate
-from ..datalake import stages
-from ..libs.stage.stage import AbstractStage
-
-
-def layout(stage_name=None, schema_name=None, table_name=None):
-    stage = stages[stage_name]
-    df = stage.read(table=table_name, schema=schema_name)
-    return html.Div([
-        dcc.Store(id="table_backup"),
-        html.Div([
-            html.H2([
-                dcc.Link(
-                    f"{stage.name}",
-                    href=f"/stage/{stage.name}",
-                    className="hover:underline"
-                ),
-                html.Span(" > "),
-                dcc.Link(
-                    f"{schema_name}",
-                    href=f"/stg/{stage.name}/schema/{schema_name}",
-                    className="hover:underline"
-                ),
-                html.Span(" > "),
-                html.Span(table_name),
-            ],
-            className="text-2xl"
-            ),
-            html.Div([
-                html.Button(
-                    "Editer",
-                    id="btn_edit",
-                    className="rounded border px-2 py-1",
-                    style={"display": "block"}
-                ),
-                html.Button(
-                    "Sauver",
-                    id="btn_save",
-                    className="rounded border px-2 py-1 border-green-500 hover:bg-green-500",
-                    style={"display": "none"}
-                ),
-                html.Button(
-                    "Annuler",
-                    id="btn_cancel",
-                    className="rounded border px-2 py-1 border-red-500 hover:bg-red-500",
-                    style={"display": "none"}
-                ),
-            ],
-            className="flex flex-row space-x-2",
-            id="toolbar"
-            ),
-        ],
-        className="flex flex-row justify-between p-4"
-        ),
-        html.Div([
-            html.Div([
-                dash_table.DataTable(
-                    id="datatable",
-                    data=df.to_dict('records'),
-                    columns=[{"name": i, "id": i} for i in df.columns],
-                    filter_action="native",
-                    sort_action="native",
-                    sort_mode="multi",
-                    editable=False
-                )
-            ])
-        ],
-     className="overflow-y-auto"
-     ),
-    ],
-    className="p-2"
-    )
-
-
-@callback(
-    Output("datatable", 'editable', allow_duplicate=True),
-    Output("table_backup", 'data'),
-    Input("btn_edit", "n_clicks"),
-    State("datatable", 'data'),
-    prevent_initial_call=True
-)
-def activate_editable(n_clicks, df_src):
-    if n_clicks is None:
-        raise PreventUpdate
-    if n_clicks > 0:
-        df_backup = df_src.copy()
-        return True, df_backup
-    raise PreventUpdate
-
-@callback(
-    Output("datatable", 'editable'),
-    Output("datatable", 'data'),
-    Input("btn_cancel", "n_clicks"),
-    State("table_backup", 'data'),
-    prevent_initial_call=True
-)
-def cancel_modifications(n_clicks, data):
-    if n_clicks is None:
-        raise PreventUpdate
-    if n_clicks > 0 and data is not None:
-        return False, data.copy()
-    raise PreventUpdate
-
-# @callback(
-#     Output("datatable", 'editable'),
-#     Input("btn_save", "n_clicks"),
-#     State("datatable", 'editable'),
-# )
-# def save_modifications(n_clicks, editable):
-#     if n_clicks is None:
-#         raise PreventUpdate
-#     if n_clicks > 0:
-#         return not editable
-#     return editable
-
-@callback(
-    Output("btn_edit", "style"),
-    Output("btn_save", "style"),
-    Output("btn_cancel", "style"),
-    Input("datatable", "editable"),
-)
-def toolbar(editable):
-    if editable:
-        return {"display": "none"}, {"display": "block"}, {"display": "block"}
-    return {"display": "block"}, {"display": "none"}, {"display": "none"}