refact: rename stage to repository

dashboard/app.py

@@ -1,62 +1,64 @@
 import dash
-from dash import Dash, html, dcc
+from dash import Dash, dcc, html
-from .pages import home, config, stage, schema, table
-from .datalake import stages
 
-external_scripts = [
+from .datalake import stages
-    {'src': 'https://cdn.tailwindcss.com'}
+from .pages import config, home, repository, schema, table
-]
+
+external_scripts = [{"src": "https://cdn.tailwindcss.com"}]
 # external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}]
 
-app = Dash(__name__,
+app = Dash(
+    __name__,
     use_pages=True,
     external_scripts=external_scripts,
     suppress_callback_exceptions=True,
-           )
+)
 app.scripts.config.serve_locally = True
 dash.register_page(
     home.__name__,
-    path='/', 
+    path="/",
     layout=home.layout,
 )
+dash.register_page(config.__name__, path="/config", layout=config.layout)
 dash.register_page(
-    config.__name__, 
+    repository.__name__,
-    path='/config', 
+    path_template="/repository/<repository_name>",
-    layout=config.layout
+    layout=repository.layout_factory(stages),
-)
-dash.register_page(
-    stage.__name__, 
-    path_template='/stage/<stage_name>', 
-    layout=stage.layout
 )
 dash.register_page(
     schema.__name__,
-    path_template='/stg/<stage_name>/schema/<schema_name>',
+    path_template="/stg/<repository_name>/schema/<schema_name>",
-    layout=schema.layout
+    layout=schema.layout_factory(stages),
 )
 dash.register_page(
     table.__name__,
-    path_template='/stg/<stage_name>/schm/<schema_name>/table/<table_name>',
+    path_template="/stg/<repository_name>/schm/<schema_name>/table/<table_name>",
-    layout=table.layout_factory(stages)
+    layout=table.layout_factory(stages),
 )
 table.callback_factory(app)
 
-app.layout = html.Div([
+app.layout = html.Div(
-    html.Div([
+    [
+        html.Div(
+            [
                 dcc.Link(
-            html.H1('Plesna', ),
+                    html.H1(
+                        "Plesna",
+                    ),
                     href="/",
-            className="text-4xl p-4 text-center grow align-baseline"
+                    className="text-4xl p-4 text-center grow align-baseline",
                 ),
-        dcc.Link("Config",
+                dcc.Link(
+                    "Config",
                     href="/config",
-                 className="flex-none hover:bg-amber-100 p-4 align-middle"
+                    className="flex-none hover:bg-amber-100 p-4 align-middle",
-                 )
-    ],
-        className="bg-amber-300 flex flex-row shadow"
                 ),
-    dash.page_container
+            ],
-])
+            className="bg-amber-300 flex flex-row shadow",
+        ),
+        dash.page_container,
+    ]
+)
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run(debug=True)

dashboard/components/lists.py

@@ -1,7 +1,9 @@
-from ..libs.stage.stage import AbstractStage
+from dash import dcc, html
-from dash import html, dcc
 
-def html_list_schema(stage:AbstractStage, with_tables=True):
+from ..libs.repository.repository import AbstractRepository
+
+
+def html_list_schema(stage:AbstractRepository, with_tables=True):
     """ Build html list of schema in stage """
     ul_classes = "ml-2"
     schema_baseurl = f"/stg/{stage.name}/schema/"
@@ -36,7 +38,7 @@ def html_list_schema(stage:AbstractStage, with_tables=True):
     )
 
 
-def html_list_table(stage:AbstractStage, schema:str):
+def html_list_table(stage:AbstractRepository, schema:str):
     """ Build html list of table in stage """
     table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/"
     return html.Ul(

dashboard/datalake.py

@@ -1,13 +1,14 @@
-from .libs.stage.fs_stage import FSStage
 from dotenv import dotenv_values
 
+from .libs.repository.fs_repository import FSRepository
+
 env = {
     **dotenv_values(".env"),
 }
 
 stages = {
-    "raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
+    "raw": FSRepository("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
-    "staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
+    "staging": FSRepository("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
-    "gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
+    "gold": FSRepository("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
-    "mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
+    "mart": FSRepository("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
 }

dashboard/libs/repository/fs_repository.py

@@ -1,8 +1,11 @@
-from .stage import AbstractStage
 from pathlib import Path
+
 import pandas as pd
 
-class FSStage(AbstractStage):
+from .repository import AbstractRepository
+
+
+class FSRepository(AbstractRepository):
     def __init__(self, name, basepath, metadata_engine=None):
         self.name = name
 

dashboard/libs/repository/repository.py

@@ -2,7 +2,7 @@ import abc
 from .metadata import AbstractMetadataEngine
 
 
-class AbstractStage(abc.ABC):
+class AbstractRepository(abc.ABC):
     metadata_engine = AbstractMetadataEngine
 
     @abc.abstractmethod

dashboard/pages/home.py

@@ -1,7 +1,7 @@
-from dash import html, dcc
+from dash import dcc, html
-from ..datalake import stages
-from ..components.lists import html_list_schema
 
+from ..components.lists import html_list_schema
+from ..datalake import stages
 
 layout = html.Div([
     html.Div(children=[

dashboard/pages/repository.py

@@ -0,0 +1,18 @@
+from dash import html
+
+from ..components.lists import html_list_schema
+from ..libs.repository.repository import AbstractRepository
+
+
+def layout_factory(repositories: dict[str, AbstractRepository]):
+    def layout(repository_name: str = ""):
+        repository = repositories[repository_name]
+        return html.Div(
+            [
+                html.H2(f"{repository.name}", className="text-2xl p-4 py-2"),
+                html_list_schema(repository),
+            ],
+            className="flex flex-col",
+        )
+
+    return layout

dashboard/pages/schema.py

@@ -1,24 +1,28 @@
-from dash import html, dcc
+from dash import dcc, html
-from ..datalake import stages
+
-from ..libs.stage.stage import AbstractStage
+from ..libs.repository.repository import AbstractRepository
 
 
-def layout(stage_name=None, schema_name=None):
+def layout_factory(repositories: dict[str, AbstractRepository]):
-    stage = stages[stage_name]
+    def layout(repository_name: str = "", schema_name: str = ""):
-    return html.Div([
+        repository = repositories[repository_name]
-        html.H2([
+        return html.Div(
+            [
+                html.H2(
+                    [
                         dcc.Link(
-                f"{stage.name}",
+                            f"{repository.name}",
-                href=f"/stage/{stage.name}",
+                            href=f"/repository/{repository.name}",
-                className="hover:underline"
+                            className="hover:underline",
                         ),
                         html.Span(" > "),
                         html.Span(
                             f"{schema_name}",
                         ),
                     ],
-            className="text-2xl p-4 py-2"
+                    className="text-2xl p-4 py-2",
-
                 ),
-    ])
+            ]
+        )
 
+    return layout

dashboard/pages/stage.py

@@ -1,18 +0,0 @@
-from dash import html, dcc
-from ..datalake import stages
-from ..libs.stage.stage import AbstractStage
-from ..components.lists import html_list_schema
-
-
-def layout(stage_name=None):
-    stage = stages[stage_name]
-    return html.Div([
-
-        html.H2(
-            f"{stage.name}",
-            className="text-2xl p-4 py-2"
-        ),
-            html_list_schema(stage)
-        ],
-        className = "flex flex-col"
-    )

dashboard/pages/table.py

@@ -1,24 +1,26 @@
-from dash import html, dcc, dash_table, Input, Output, State
+from dash import Input, Output, State, dash_table, dcc, html
 from dash.exceptions import PreventUpdate
-from ..libs.stage.stage import AbstractStage
 
-def layout_factory(stages: list[AbstractStage]):
+from ..libs.repository.repository import AbstractRepository
-    def layout(stage_name=None, schema_name=None, table_name=None):
+
-        stage = stages[stage_name]
+
-        df = stage.read(table=table_name, schema=schema_name)
+def layout_factory(repositories: dict[str,AbstractRepository]):
+    def layout(repository_name:str="", schema_name:str="", table_name:str=""):
+        repository = repositories[repository_name]
+        df = repository.read(table=table_name, schema=schema_name)
         return html.Div([
             dcc.Store(id="table_backup"),
             html.Div([
                 html.H2([
                     dcc.Link(
-                        f"{stage.name}",
+                        f"{repository.name}",
-                        href=f"/stage/{stage.name}",
+                        href=f"/repository/{repository.name}",
                         className="hover:underline"
                     ),
                     html.Span(" > "),
                     dcc.Link(
                         f"{schema_name}",
-                        href=f"/stg/{stage.name}/schema/{schema_name}",
+                        href=f"/stg/{repository.name}/schema/{schema_name}",
                         className="hover:underline"
                     ),
                     html.Span(" > "),