From 2de0e5ef5caa91f3b068ec9a140b98a3e0f3e0cb Mon Sep 17 00:00:00 2001 From: Bertrand Benjamin Date: Wed, 7 Aug 2024 11:39:33 +0200 Subject: [PATCH] refact: rename stage to repository --- dashboard/app.py | 86 ++++++++++--------- dashboard/components/lists.py | 10 ++- dashboard/datalake.py | 11 +-- .../fs_repository.py} | 7 +- .../libs/{stage => repository}/metadata.py | 0 .../stage.py => repository/repository.py} | 2 +- dashboard/pages/home.py | 6 +- dashboard/pages/repository.py | 18 ++++ dashboard/pages/schema.py | 46 +++++----- dashboard/pages/stage.py | 18 ---- dashboard/pages/table.py | 20 +++-- 11 files changed, 119 insertions(+), 105 deletions(-) rename dashboard/libs/{stage/fs_stage.py => repository/fs_repository.py} (95%) rename dashboard/libs/{stage => repository}/metadata.py (100%) rename dashboard/libs/{stage/stage.py => repository/repository.py} (95%) create mode 100644 dashboard/pages/repository.py delete mode 100644 dashboard/pages/stage.py diff --git a/dashboard/app.py b/dashboard/app.py index 7831ed4..12d4714 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -1,62 +1,64 @@ import dash -from dash import Dash, html, dcc -from .pages import home, config, stage, schema, table -from .datalake import stages +from dash import Dash, dcc, html -external_scripts = [ - {'src': 'https://cdn.tailwindcss.com'} -] +from .datalake import stages +from .pages import config, home, repository, schema, table + +external_scripts = [{"src": "https://cdn.tailwindcss.com"}] # external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}] -app = Dash(__name__, - use_pages=True, - external_scripts=external_scripts, - suppress_callback_exceptions=True, - ) +app = Dash( + __name__, + use_pages=True, + external_scripts=external_scripts, + suppress_callback_exceptions=True, +) app.scripts.config.serve_locally = True dash.register_page( - home.__name__, - path='/', + home.__name__, + path="/", layout=home.layout, ) +dash.register_page(config.__name__, path="/config", layout=config.layout) dash.register_page( - config.__name__, - path='/config', - layout=config.layout + repository.__name__, + path_template="/repository/", + layout=repository.layout_factory(stages), ) dash.register_page( - stage.__name__, - path_template='/stage/', - layout=stage.layout -) -dash.register_page( - schema.__name__, - path_template='/stg//schema/', - layout=schema.layout + schema.__name__, + path_template="/stg//schema/", + layout=schema.layout_factory(stages), ) dash.register_page( table.__name__, - path_template='/stg//schm//table/', - layout=table.layout_factory(stages) + path_template="/stg//schm//table/", + layout=table.layout_factory(stages), ) table.callback_factory(app) -app.layout = html.Div([ - html.Div([ - dcc.Link( - html.H1('Plesna', ), - href="/", - className="text-4xl p-4 text-center grow align-baseline" +app.layout = html.Div( + [ + html.Div( + [ + dcc.Link( + html.H1( + "Plesna", + ), + href="/", + className="text-4xl p-4 text-center grow align-baseline", + ), + dcc.Link( + "Config", + href="/config", + className="flex-none hover:bg-amber-100 p-4 align-middle", + ), + ], + className="bg-amber-300 flex flex-row shadow", ), - dcc.Link("Config", - href="/config", - className="flex-none hover:bg-amber-100 p-4 align-middle" - ) - ], - className="bg-amber-300 flex flex-row shadow" - ), - dash.page_container -]) + dash.page_container, + ] +) -if __name__ == '__main__': +if __name__ == "__main__": app.run(debug=True) diff --git a/dashboard/components/lists.py b/dashboard/components/lists.py index 88235c4..7be9f33 100644 --- a/dashboard/components/lists.py +++ b/dashboard/components/lists.py @@ -1,7 +1,9 @@ -from ..libs.stage.stage import AbstractStage -from dash import html, dcc +from dash import dcc, html -def html_list_schema(stage:AbstractStage, with_tables=True): +from ..libs.repository.repository import AbstractRepository + + +def html_list_schema(stage:AbstractRepository, with_tables=True): """ Build html list of schema in stage """ ul_classes = "ml-2" schema_baseurl = f"/stg/{stage.name}/schema/" @@ -36,7 +38,7 @@ def html_list_schema(stage:AbstractStage, with_tables=True): ) -def html_list_table(stage:AbstractStage, schema:str): +def html_list_table(stage:AbstractRepository, schema:str): """ Build html list of table in stage """ table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/" return html.Ul( diff --git a/dashboard/datalake.py b/dashboard/datalake.py index 346657e..92fa911 100644 --- a/dashboard/datalake.py +++ b/dashboard/datalake.py @@ -1,13 +1,14 @@ -from .libs.stage.fs_stage import FSStage from dotenv import dotenv_values +from .libs.repository.fs_repository import FSRepository + env = { **dotenv_values(".env"), } stages = { - "raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"), - "staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"), - "gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"), - "mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"), + "raw": FSRepository("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"), + "staging": FSRepository("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"), + "gold": FSRepository("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"), + "mart": FSRepository("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"), } diff --git a/dashboard/libs/stage/fs_stage.py b/dashboard/libs/repository/fs_repository.py similarity index 95% rename from dashboard/libs/stage/fs_stage.py rename to dashboard/libs/repository/fs_repository.py index a22067b..3c4eee8 100644 --- a/dashboard/libs/stage/fs_stage.py +++ b/dashboard/libs/repository/fs_repository.py @@ -1,8 +1,11 @@ -from .stage import AbstractStage from pathlib import Path + import pandas as pd -class FSStage(AbstractStage): +from .repository import AbstractRepository + + +class FSRepository(AbstractRepository): def __init__(self, name, basepath, metadata_engine=None): self.name = name diff --git a/dashboard/libs/stage/metadata.py b/dashboard/libs/repository/metadata.py similarity index 100% rename from dashboard/libs/stage/metadata.py rename to dashboard/libs/repository/metadata.py diff --git a/dashboard/libs/stage/stage.py b/dashboard/libs/repository/repository.py similarity index 95% rename from dashboard/libs/stage/stage.py rename to dashboard/libs/repository/repository.py index 993c18d..3605de7 100644 --- a/dashboard/libs/stage/stage.py +++ b/dashboard/libs/repository/repository.py @@ -2,7 +2,7 @@ import abc from .metadata import AbstractMetadataEngine -class AbstractStage(abc.ABC): +class AbstractRepository(abc.ABC): metadata_engine = AbstractMetadataEngine @abc.abstractmethod diff --git a/dashboard/pages/home.py b/dashboard/pages/home.py index 506fd51..ae427f3 100644 --- a/dashboard/pages/home.py +++ b/dashboard/pages/home.py @@ -1,7 +1,7 @@ -from dash import html, dcc -from ..datalake import stages -from ..components.lists import html_list_schema +from dash import dcc, html +from ..components.lists import html_list_schema +from ..datalake import stages layout = html.Div([ html.Div(children=[ diff --git a/dashboard/pages/repository.py b/dashboard/pages/repository.py new file mode 100644 index 0000000..c603257 --- /dev/null +++ b/dashboard/pages/repository.py @@ -0,0 +1,18 @@ +from dash import html + +from ..components.lists import html_list_schema +from ..libs.repository.repository import AbstractRepository + + +def layout_factory(repositories: dict[str, AbstractRepository]): + def layout(repository_name: str = ""): + repository = repositories[repository_name] + return html.Div( + [ + html.H2(f"{repository.name}", className="text-2xl p-4 py-2"), + html_list_schema(repository), + ], + className="flex flex-col", + ) + + return layout diff --git a/dashboard/pages/schema.py b/dashboard/pages/schema.py index 29230e7..9c3a6eb 100644 --- a/dashboard/pages/schema.py +++ b/dashboard/pages/schema.py @@ -1,24 +1,28 @@ -from dash import html, dcc -from ..datalake import stages -from ..libs.stage.stage import AbstractStage +from dash import dcc, html + +from ..libs.repository.repository import AbstractRepository -def layout(stage_name=None, schema_name=None): - stage = stages[stage_name] - return html.Div([ - html.H2([ - dcc.Link( - f"{stage.name}", - href=f"/stage/{stage.name}", - className="hover:underline" - ), - html.Span(" > "), - html.Span( - f"{schema_name}", - ), - ], - className="text-2xl p-4 py-2" - - ), - ]) +def layout_factory(repositories: dict[str, AbstractRepository]): + def layout(repository_name: str = "", schema_name: str = ""): + repository = repositories[repository_name] + return html.Div( + [ + html.H2( + [ + dcc.Link( + f"{repository.name}", + href=f"/repository/{repository.name}", + className="hover:underline", + ), + html.Span(" > "), + html.Span( + f"{schema_name}", + ), + ], + className="text-2xl p-4 py-2", + ), + ] + ) + return layout diff --git a/dashboard/pages/stage.py b/dashboard/pages/stage.py deleted file mode 100644 index 9bb42c7..0000000 --- a/dashboard/pages/stage.py +++ /dev/null @@ -1,18 +0,0 @@ -from dash import html, dcc -from ..datalake import stages -from ..libs.stage.stage import AbstractStage -from ..components.lists import html_list_schema - - -def layout(stage_name=None): - stage = stages[stage_name] - return html.Div([ - - html.H2( - f"{stage.name}", - className="text-2xl p-4 py-2" - ), - html_list_schema(stage) - ], - className = "flex flex-col" - ) diff --git a/dashboard/pages/table.py b/dashboard/pages/table.py index ecc8d3d..9cec5e8 100644 --- a/dashboard/pages/table.py +++ b/dashboard/pages/table.py @@ -1,24 +1,26 @@ -from dash import html, dcc, dash_table, Input, Output, State +from dash import Input, Output, State, dash_table, dcc, html from dash.exceptions import PreventUpdate -from ..libs.stage.stage import AbstractStage -def layout_factory(stages: list[AbstractStage]): - def layout(stage_name=None, schema_name=None, table_name=None): - stage = stages[stage_name] - df = stage.read(table=table_name, schema=schema_name) +from ..libs.repository.repository import AbstractRepository + + +def layout_factory(repositories: dict[str,AbstractRepository]): + def layout(repository_name:str="", schema_name:str="", table_name:str=""): + repository = repositories[repository_name] + df = repository.read(table=table_name, schema=schema_name) return html.Div([ dcc.Store(id="table_backup"), html.Div([ html.H2([ dcc.Link( - f"{stage.name}", - href=f"/stage/{stage.name}", + f"{repository.name}", + href=f"/repository/{repository.name}", className="hover:underline" ), html.Span(" > "), dcc.Link( f"{schema_name}", - href=f"/stg/{stage.name}/schema/{schema_name}", + href=f"/stg/{repository.name}/schema/{schema_name}", className="hover:underline" ), html.Span(" > "),