lafrite/plesna
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refact: rename stage to repository

Browse Source
This commit is contained in:
Bertrand Benjamin 2024-08-07 11:39:33 +02:00
parent 7fb7bc6f5c
commit 2de0e5ef5c
11 changed files with 119 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
dashboard/app.py
View File

@ -1,62 +1,64 @@
import dash import dash
from dash import Dash, html, dcc from dash import Dash, dcc, html
from .pages import home, config, stage, schema, table
from .datalake import stages
external_scripts = [ from .datalake import stages
{'src': 'https://cdn.tailwindcss.com'} from .pages import config, home, repository, schema, table
]
external_scripts = [{"src": "https://cdn.tailwindcss.com"}]
# external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}] # external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}]
app = Dash(__name__, app = Dash(
use_pages=True, __name__,
external_scripts=external_scripts, use_pages=True,
suppress_callback_exceptions=True, external_scripts=external_scripts,
) suppress_callback_exceptions=True,
)
app.scripts.config.serve_locally = True app.scripts.config.serve_locally = True
dash.register_page( dash.register_page(
home.__name__, home.__name__,
path='/', path="/",
layout=home.layout, layout=home.layout,
) )
dash.register_page(config.__name__, path="/config", layout=config.layout)
dash.register_page( dash.register_page(
config.__name__, repository.__name__,
path='/config', path_template="/repository/<repository_name>",
layout=config.layout layout=repository.layout_factory(stages),
) )
dash.register_page( dash.register_page(
stage.__name__, schema.__name__,
path_template='/stage/<stage_name>', path_template="/stg/<repository_name>/schema/<schema_name>",
layout=stage.layout layout=schema.layout_factory(stages),
)
dash.register_page(
schema.__name__,
path_template='/stg/<stage_name>/schema/<schema_name>',
layout=schema.layout
) )
dash.register_page( dash.register_page(
table.__name__, table.__name__,
path_template='/stg/<stage_name>/schm/<schema_name>/table/<table_name>', path_template="/stg/<repository_name>/schm/<schema_name>/table/<table_name>",
layout=table.layout_factory(stages) layout=table.layout_factory(stages),
) )
table.callback_factory(app) table.callback_factory(app)
app.layout = html.Div([ app.layout = html.Div(
html.Div([ [
dcc.Link( html.Div(
html.H1('Plesna', ), [
href="/", dcc.Link(
className="text-4xl p-4 text-center grow align-baseline" html.H1(
"Plesna",
),
href="/",
className="text-4xl p-4 text-center grow align-baseline",
),
dcc.Link(
"Config",
href="/config",
className="flex-none hover:bg-amber-100 p-4 align-middle",
),
],
className="bg-amber-300 flex flex-row shadow",
), ),
dcc.Link("Config", dash.page_container,
href="/config", ]
className="flex-none hover:bg-amber-100 p-4 align-middle" )
)
],
className="bg-amber-300 flex flex-row shadow"
),
dash.page_container
])
if __name__ == '__main__': if __name__ == "__main__":
app.run(debug=True) app.run(debug=True)

10
dashboard/components/lists.py
View File

@ -1,7 +1,9 @@
from ..libs.stage.stage import AbstractStage from dash import dcc, html
from dash import html, dcc
def html_list_schema(stage:AbstractStage, with_tables=True): from ..libs.repository.repository import AbstractRepository
def html_list_schema(stage:AbstractRepository, with_tables=True):
""" Build html list of schema in stage """ """ Build html list of schema in stage """
ul_classes = "ml-2" ul_classes = "ml-2"
schema_baseurl = f"/stg/{stage.name}/schema/" schema_baseurl = f"/stg/{stage.name}/schema/"
@ -36,7 +38,7 @@ def html_list_schema(stage:AbstractStage, with_tables=True):
) )
def html_list_table(stage:AbstractStage, schema:str): def html_list_table(stage:AbstractRepository, schema:str):
""" Build html list of table in stage """ """ Build html list of table in stage """
table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/" table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/"
return html.Ul( return html.Ul(

11
dashboard/datalake.py
View File

@ -1,13 +1,14 @@
from .libs.stage.fs_stage import FSStage
from dotenv import dotenv_values from dotenv import dotenv_values
from .libs.repository.fs_repository import FSRepository
env = { env = {
**dotenv_values(".env"), **dotenv_values(".env"),
} }
stages = { stages = {
"raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"), "raw": FSRepository("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
"staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"), "staging": FSRepository("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
"gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"), "gold": FSRepository("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
"mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"), "mart": FSRepository("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
} }

7
dashboard/libs/stage/fs_stage.py → dashboard/libs/repository/fs_repository.py
View File

@ -1,8 +1,11 @@
from .stage import AbstractStage
from pathlib import Path from pathlib import Path
import pandas as pd import pandas as pd
class FSStage(AbstractStage): from .repository import AbstractRepository
class FSRepository(AbstractRepository):
def __init__(self, name, basepath, metadata_engine=None): def __init__(self, name, basepath, metadata_engine=None):
self.name = name self.name = name

0
dashboard/libs/stage/metadata.py → dashboard/libs/repository/metadata.py
View File

2
dashboard/libs/stage/stage.py → dashboard/libs/repository/repository.py
View File

@ -2,7 +2,7 @@ import abc
from .metadata import AbstractMetadataEngine from .metadata import AbstractMetadataEngine
class AbstractStage(abc.ABC): class AbstractRepository(abc.ABC):
metadata_engine = AbstractMetadataEngine metadata_engine = AbstractMetadataEngine
@abc.abstractmethod @abc.abstractmethod

6
dashboard/pages/home.py
View File

@ -1,7 +1,7 @@
from dash import html, dcc from dash import dcc, html
from ..datalake import stages
from ..components.lists import html_list_schema
from ..components.lists import html_list_schema
from ..datalake import stages
layout = html.Div([ layout = html.Div([
html.Div(children=[ html.Div(children=[

18
dashboard/pages/repository.py Normal file
View File

@ -0,0 +1,18 @@
from dash import html
from ..components.lists import html_list_schema
from ..libs.repository.repository import AbstractRepository
def layout_factory(repositories: dict[str, AbstractRepository]):
def layout(repository_name: str = ""):
repository = repositories[repository_name]
return html.Div(
[
html.H2(f"{repository.name}", className="text-2xl p-4 py-2"),
html_list_schema(repository),
],
className="flex flex-col",
)
return layout

46
dashboard/pages/schema.py
View File

@ -1,24 +1,28 @@
from dash import html, dcc from dash import dcc, html
from ..datalake import stages
from ..libs.stage.stage import AbstractStage from ..libs.repository.repository import AbstractRepository
def layout(stage_name=None, schema_name=None): def layout_factory(repositories: dict[str, AbstractRepository]):
stage = stages[stage_name] def layout(repository_name: str = "", schema_name: str = ""):
return html.Div([ repository = repositories[repository_name]
html.H2([ return html.Div(
dcc.Link( [
f"{stage.name}", html.H2(
href=f"/stage/{stage.name}", [
className="hover:underline" dcc.Link(
), f"{repository.name}",
html.Span(" > "), href=f"/repository/{repository.name}",
html.Span( className="hover:underline",
f"{schema_name}", ),
), html.Span(" > "),
], html.Span(
className="text-2xl p-4 py-2" f"{schema_name}",
),
), ],
]) className="text-2xl p-4 py-2",
),
]
)
return layout

18
dashboard/pages/stage.py
View File

@ -1,18 +0,0 @@
from dash import html, dcc
from ..datalake import stages
from ..libs.stage.stage import AbstractStage
from ..components.lists import html_list_schema
def layout(stage_name=None):
stage = stages[stage_name]
return html.Div([
html.H2(
f"{stage.name}",
className="text-2xl p-4 py-2"
),
html_list_schema(stage)
],
className = "flex flex-col"
)

20
dashboard/pages/table.py
View File

@ -1,24 +1,26 @@
from dash import html, dcc, dash_table, Input, Output, State from dash import Input, Output, State, dash_table, dcc, html
from dash.exceptions import PreventUpdate from dash.exceptions import PreventUpdate
from ..libs.stage.stage import AbstractStage
def layout_factory(stages: list[AbstractStage]): from ..libs.repository.repository import AbstractRepository
def layout(stage_name=None, schema_name=None, table_name=None):
stage = stages[stage_name]
df = stage.read(table=table_name, schema=schema_name) def layout_factory(repositories: dict[str,AbstractRepository]):
def layout(repository_name:str="", schema_name:str="", table_name:str=""):
repository = repositories[repository_name]
df = repository.read(table=table_name, schema=schema_name)
return html.Div([ return html.Div([
dcc.Store(id="table_backup"), dcc.Store(id="table_backup"),
html.Div([ html.Div([
html.H2([ html.H2([
dcc.Link( dcc.Link(
f"{stage.name}", f"{repository.name}",
href=f"/stage/{stage.name}", href=f"/repository/{repository.name}",
className="hover:underline" className="hover:underline"
), ),
html.Span(" > "), html.Span(" > "),
dcc.Link( dcc.Link(
f"{schema_name}", f"{schema_name}",
href=f"/stg/{stage.name}/schema/{schema_name}", href=f"/stg/{repository.name}/schema/{schema_name}",
className="hover:underline" className="hover:underline"
), ),
html.Span(" > "), html.Span(" > "),