lafrite/plesna
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refact: rename stage to repository

Browse Source
This commit is contained in:
Bertrand Benjamin 2024-08-07 11:39:33 +02:00
parent 7fb7bc6f5c
commit 2de0e5ef5c
11 changed files with 119 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
dashboard/app.py
View File

@ -1,14 +1,14 @@
import dash
from dash import Dash, html, dcc
from .pages import home, config, stage, schema, table
from .datalake import stages
from dash import Dash, dcc, html
external_scripts = [
{'src': 'https://cdn.tailwindcss.com'}
]
from .datalake import stages
from .pages import config, home, repository, schema, table
external_scripts = [{"src": "https://cdn.tailwindcss.com"}]
# external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}]
app = Dash(__name__,
app = Dash(
__name__,
use_pages=True,
external_scripts=external_scripts,
suppress_callback_exceptions=True,
@ -16,47 +16,49 @@ app = Dash(__name__,
app.scripts.config.serve_locally = True
dash.register_page(
home.__name__,
path='/',
path="/",
layout=home.layout,
)
dash.register_page(config.__name__, path="/config", layout=config.layout)
dash.register_page(
config.__name__,
path='/config',
layout=config.layout
)
dash.register_page(
stage.__name__,
path_template='/stage/<stage_name>',
layout=stage.layout
repository.__name__,
path_template="/repository/<repository_name>",
layout=repository.layout_factory(stages),
)
dash.register_page(
schema.__name__,
path_template='/stg/<stage_name>/schema/<schema_name>',
layout=schema.layout
path_template="/stg/<repository_name>/schema/<schema_name>",
layout=schema.layout_factory(stages),
)
dash.register_page(
table.__name__,
path_template='/stg/<stage_name>/schm/<schema_name>/table/<table_name>',
layout=table.layout_factory(stages)
path_template="/stg/<repository_name>/schm/<schema_name>/table/<table_name>",
layout=table.layout_factory(stages),
)
table.callback_factory(app)
app.layout = html.Div([
html.Div([
app.layout = html.Div(
[
html.Div(
[
dcc.Link(
html.H1('Plesna', ),
html.H1(
"Plesna",
),
href="/",
className="text-4xl p-4 text-center grow align-baseline"
className="text-4xl p-4 text-center grow align-baseline",
),
dcc.Link("Config",
dcc.Link(
"Config",
href="/config",
className="flex-none hover:bg-amber-100 p-4 align-middle"
)
],
className="bg-amber-300 flex flex-row shadow"
className="flex-none hover:bg-amber-100 p-4 align-middle",
),
dash.page_container
])
],
className="bg-amber-300 flex flex-row shadow",
),
dash.page_container,
]
)
if __name__ == '__main__':
if __name__ == "__main__":
app.run(debug=True)

10
dashboard/components/lists.py
View File

@ -1,7 +1,9 @@
from ..libs.stage.stage import AbstractStage
from dash import html, dcc
from dash import dcc, html
def html_list_schema(stage:AbstractStage, with_tables=True):
from ..libs.repository.repository import AbstractRepository
def html_list_schema(stage:AbstractRepository, with_tables=True):
""" Build html list of schema in stage """
ul_classes = "ml-2"
schema_baseurl = f"/stg/{stage.name}/schema/"
@ -36,7 +38,7 @@ def html_list_schema(stage:AbstractStage, with_tables=True):
)
def html_list_table(stage:AbstractStage, schema:str):
def html_list_table(stage:AbstractRepository, schema:str):
""" Build html list of table in stage """
table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/"
return html.Ul(

11
dashboard/datalake.py
View File

@ -1,13 +1,14 @@
from .libs.stage.fs_stage import FSStage
from dotenv import dotenv_values
from .libs.repository.fs_repository import FSRepository
env = {
**dotenv_values(".env"),
}
stages = {
"raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
"staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
"gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
"mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
"raw": FSRepository("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
"staging": FSRepository("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
"gold": FSRepository("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
"mart": FSRepository("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
}

7
dashboard/libs/stage/fs_stage.py → dashboard/libs/repository/fs_repository.py
View File

@ -1,8 +1,11 @@
from .stage import AbstractStage
from pathlib import Path
import pandas as pd
class FSStage(AbstractStage):
from .repository import AbstractRepository
class FSRepository(AbstractRepository):
def __init__(self, name, basepath, metadata_engine=None):
self.name = name

0
dashboard/libs/stage/metadata.py → dashboard/libs/repository/metadata.py
View File

2
dashboard/libs/stage/stage.py → dashboard/libs/repository/repository.py
View File

@ -2,7 +2,7 @@ import abc
from .metadata import AbstractMetadataEngine
class AbstractStage(abc.ABC):
class AbstractRepository(abc.ABC):
metadata_engine = AbstractMetadataEngine
@abc.abstractmethod

6
dashboard/pages/home.py
View File

@ -1,7 +1,7 @@
from dash import html, dcc
from ..datalake import stages
from ..components.lists import html_list_schema
from dash import dcc, html
from ..components.lists import html_list_schema
from ..datalake import stages
layout = html.Div([
html.Div(children=[

18
dashboard/pages/repository.py Normal file
View File

@ -0,0 +1,18 @@
from dash import html
from ..components.lists import html_list_schema
from ..libs.repository.repository import AbstractRepository
def layout_factory(repositories: dict[str, AbstractRepository]):
def layout(repository_name: str = ""):
repository = repositories[repository_name]
return html.Div(
[
html.H2(f"{repository.name}", className="text-2xl p-4 py-2"),
html_list_schema(repository),
],
className="flex flex-col",
)
return layout

30
dashboard/pages/schema.py
View File

@ -1,24 +1,28 @@
from dash import html, dcc
from ..datalake import stages
from ..libs.stage.stage import AbstractStage
from dash import dcc, html
from ..libs.repository.repository import AbstractRepository
def layout(stage_name=None, schema_name=None):
stage = stages[stage_name]
return html.Div([
html.H2([
def layout_factory(repositories: dict[str, AbstractRepository]):
def layout(repository_name: str = "", schema_name: str = ""):
repository = repositories[repository_name]
return html.Div(
[
html.H2(
[
dcc.Link(
f"{stage.name}",
href=f"/stage/{stage.name}",
className="hover:underline"
f"{repository.name}",
href=f"/repository/{repository.name}",
className="hover:underline",
),
html.Span(" > "),
html.Span(
f"{schema_name}",
),
],
className="text-2xl p-4 py-2"
className="text-2xl p-4 py-2",
),
])
]
)
return layout

18
dashboard/pages/stage.py
View File

@ -1,18 +0,0 @@
from dash import html, dcc
from ..datalake import stages
from ..libs.stage.stage import AbstractStage
from ..components.lists import html_list_schema
def layout(stage_name=None):
stage = stages[stage_name]
return html.Div([
html.H2(
f"{stage.name}",
className="text-2xl p-4 py-2"
),
html_list_schema(stage)
],
className = "flex flex-col"
)

20
dashboard/pages/table.py
View File

@ -1,24 +1,26 @@
from dash import html, dcc, dash_table, Input, Output, State
from dash import Input, Output, State, dash_table, dcc, html
from dash.exceptions import PreventUpdate
from ..libs.stage.stage import AbstractStage
def layout_factory(stages: list[AbstractStage]):
def layout(stage_name=None, schema_name=None, table_name=None):
stage = stages[stage_name]
df = stage.read(table=table_name, schema=schema_name)
from ..libs.repository.repository import AbstractRepository
def layout_factory(repositories: dict[str,AbstractRepository]):
def layout(repository_name:str="", schema_name:str="", table_name:str=""):
repository = repositories[repository_name]
df = repository.read(table=table_name, schema=schema_name)
return html.Div([
dcc.Store(id="table_backup"),
html.Div([
html.H2([
dcc.Link(
f"{stage.name}",
href=f"/stage/{stage.name}",
f"{repository.name}",
href=f"/repository/{repository.name}",
className="hover:underline"
),
html.Span(" > "),
dcc.Link(
f"{schema_name}",
href=f"/stg/{stage.name}/schema/{schema_name}",
href=f"/stg/{repository.name}/schema/{schema_name}",
className="hover:underline"
),
html.Span(" > "),