Compare commits

..

No commits in common. "612df0a8ebf6b4372de772acbee8fd10955aa042" and "1ed6ed43ed98b296201d9c423030ca8193b30389" have entirely different histories.

15 changed files with 0 additions and 474 deletions

View File

@ -1,60 +0,0 @@
import dash
from dash import Dash, html, dcc
from .pages import home, config, stage, schema, table
external_scripts = [
{'src': 'https://cdn.tailwindcss.com'}
]
# external_script = ["https://tailwindcss.com/", {"src": "https://cdn.tailwindcss.com"}]
app = Dash(__name__,
use_pages=True,
external_scripts=external_scripts,
suppress_callback_exceptions=True,
)
app.scripts.config.serve_locally = True
dash.register_page(
home.__name__,
path='/',
layout=home.layout,
)
dash.register_page(
config.__name__,
path='/config',
layout=config.layout
)
dash.register_page(
stage.__name__,
path_template='/stage/<stage_name>',
layout=stage.layout
)
dash.register_page(
schema.__name__,
path_template='/stg/<stage_name>/schema/<schema_name>',
layout=schema.layout
)
dash.register_page(
table.__name__,
path_template='/stg/<stage_name>/schm/<schema_name>/table/<table_name>',
layout=table.layout
)
app.layout = html.Div([
html.Div([
dcc.Link(
html.H1('Plesna', ),
href="/",
className="text-4xl p-4 text-center grow align-baseline"
),
dcc.Link("Config",
href="/config",
className="flex-none hover:bg-amber-100 p-4 align-middle"
)
],
className="bg-amber-300 flex flex-row shadow"
),
dash.page_container
])
if __name__ == '__main__':
app.run(debug=True)

View File

@ -1,55 +0,0 @@
from ..libs.stage.stage import AbstractStage
from dash import html, dcc
def html_list_schema(stage:AbstractStage, with_tables=True):
""" Build html list of schema in stage """
ul_classes = "ml-2"
schema_baseurl = f"/stg/{stage.name}/schema/"
if with_tables:
return html.Ul(
[
html.Li(
children = [
dcc.Link(
schema,
href=schema_baseurl + schema,
className="text-lg hover:underline"
),
html_list_table(stage, schema)
],
className=""
) for schema in stage.schemas()
],
className=ul_classes
)
return html.Ul(
[
html.Li(
dcc.Link(
schema,
href=schema_baseurl + schema,
className="text-lg hover:underline"
),
) for schema in stage.schemas()
],
className=ul_classes
)
def html_list_table(stage:AbstractStage, schema:str):
""" Build html list of table in stage """
table_baseurl = f"/stg/{stage.name}/schm/{schema}/table/"
return html.Ul(
[
html.Li(
dcc.Link(
table,
href=table_baseurl + table,
className="hover:underline"
),
) for table in stage.tables(schema=schema)
],
className="ml-4"
)

View File

@ -1,13 +0,0 @@
from .libs.stage.fs_stage import FSStage
from dotenv import dotenv_values
env = {
**dotenv_values(".env"),
}
stages = {
"raw": FSStage("raw", f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
"staging": FSStage("staging", f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
"gold": FSStage("gold", f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
"mart": FSStage("mart", f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
}

View File

@ -1,35 +0,0 @@
from .schema import AbstractSchema
from pathlib import Path
class FSSchema(AbstractSchema):
def __init__(self, basepath, metadata_engine=None):
self.basepath = basepath
self._metadata_engine = metadata_engine
def ls(self, dir, only_files=True):
dirpath = Path(dir)
if only_files:
return [f for f in dirpath.iterdir() if f.is_dir()]
return [f for f in dirpath.iterdir()]
def tables(self, dir, only_files=True):
dirpath = Path(dir)
if only_files:
return [f for f in dirpath.iterdir() if f.is_dir()]
return [f for f in dirpath.iterdir()]
def info(self, path):
path = Path(path)
pass
def read(self, path):
path = Path(path)
pass
def write(self, path, content):
path = Path(path)
pass
def delete(self, path):
path = Path(path)
pass

View File

@ -1,61 +0,0 @@
from .stage import AbstractStage
from pathlib import Path
import pandas as pd
class FSStage(AbstractStage):
def __init__(self, name, basepath, metadata_engine=None):
self.name = name
self.basepath = Path(basepath)
self._metadata_engine = metadata_engine
def ls(self, dir, only_files=False, only_directories=False, recursive=False) -> list[str]:
dirpath = Path(dir)
if only_files:
return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if not f.is_dir()]
if only_directories:
if recursive:
return [str(f[0].relative_to(dirpath)) for f in dirpath.walk()]
return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if f.is_dir()]
return [str(f.relative_to(dirpath)) for f in dirpath.iterdir()]
def schemas(self, recursive=True) -> list[str]:
dirpath = self.basepath
return self.ls(dirpath, only_directories=True, recursive=True)
def tables(self, schema:str) -> list[str]:
dirpath = self.basepath / schema
return self.ls(dirpath, only_files=True)
def build_table_path(self, table:str, schema:str):
table_path = self.basepath
if schema == '.':
return table_path / table
return table_path / schema / table
def info(self, table:str, schema:str='.'):
table_path = self.build_table_path(table, schema)
pass
def read(self, table:str, schema:str='.', read_options={}):
table_path = self.build_table_path(table, schema)
extension = table_path.suffix
if extension == '.csv':
return pd.read_csv(table_path, **read_options)
if extension == '.xlsx':
return pd.read_excel(table_path, **read_options)
raise ValueError("Can't open the table")
def write(self, table:str, content, schema:str='.'):
table_path = self.build_table_path(table, schema)
pass
def delete(self, table:str, schema:str='.'):
table_path = self.build_table_path(table, schema)
pass

View File

@ -1,5 +0,0 @@
from abc import ABC
class AbstractMetadataEngine(ABC):
pass

View File

@ -1,36 +0,0 @@
import abc
from .metadata import AbstractMetadataEngine
class AbstractStage(abc.ABC):
metadata_engine = AbstractMetadataEngine
@abc.abstractmethod
def schemas():
""" List schemas """
raise NotImplementedError
@abc.abstractmethod
def tables(schema):
""" List table in schema"""
raise NotImplementedError
@abc.abstractmethod
def info(self, path):
""" Get infos about a file"""
raise NotImplementedError
@abc.abstractmethod
def read(self, path):
""" Get content of a file"""
raise NotImplementedError
@abc.abstractmethod
def write(self, path, content):
""" Write content into the file"""
raise NotImplementedError
@abc.abstractmethod
def delete(self, path):
""" Delete the file """
raise NotImplementedError

View File

@ -1,14 +0,0 @@
from dash import html
from dotenv import dotenv_values
import os
env = {
**dotenv_values(".env"),
**os.environ,
}
layout = html.Div([
html.H1('This is our Config page'),
html.Ul(children = [html.Li(f"{k} = {v}") for k,v in env.items()]),
])

View File

@ -1,27 +0,0 @@
from dash import html, dcc
from ..datalake import stages
from ..components.lists import html_list_schema
layout = html.Div([
html.Div(children=[
html.Ul(
children=[
html.Li(
children=[
dcc.Link(
stagename,
href=f"/stage/{stagename}",
className="text-2xl text-center p-2 bg-amber-100 rounded shadow"
),
html_list_schema(stage)
],
className="flex-1 bg-gray-100 rounded flex flex-col shadow"
) for stagename, stage in stages.items()
],
className="flex flex-row space-x-2"
)
],
className="w-full mt-4 px-2"
),
])

View File

@ -1,24 +0,0 @@
from dash import html, dcc
from ..datalake import stages
from ..libs.stage.stage import AbstractStage
def layout(stage_name=None, schema_name=None):
stage = stages[stage_name]
return html.Div([
html.H2([
dcc.Link(
f"{stage.name}",
href=f"/stage/{stage.name}",
className="hover:underline"
),
html.Span(" > "),
html.Span(
f"{schema_name}",
),
],
className="text-2xl p-4 py-2"
),
])

View File

@ -1,18 +0,0 @@
from dash import html, dcc
from ..datalake import stages
from ..libs.stage.stage import AbstractStage
from ..components.lists import html_list_schema
def layout(stage_name=None):
stage = stages[stage_name]
return html.Div([
html.H2(
f"{stage.name}",
className="text-2xl p-4 py-2"
),
html_list_schema(stage)
],
className = "flex flex-col"
)

View File

@ -1,126 +0,0 @@
from dash import html, dcc, dash_table, callback, Input, Output, State
from dash.exceptions import PreventUpdate
from ..datalake import stages
from ..libs.stage.stage import AbstractStage
def layout(stage_name=None, schema_name=None, table_name=None):
stage = stages[stage_name]
df = stage.read(table=table_name, schema=schema_name)
return html.Div([
dcc.Store(id="table_backup"),
html.Div([
html.H2([
dcc.Link(
f"{stage.name}",
href=f"/stage/{stage.name}",
className="hover:underline"
),
html.Span(" > "),
dcc.Link(
f"{schema_name}",
href=f"/stg/{stage.name}/schema/{schema_name}",
className="hover:underline"
),
html.Span(" > "),
html.Span(table_name),
],
className="text-2xl"
),
html.Div([
html.Button(
"Editer",
id="btn_edit",
className="rounded border px-2 py-1",
style={"display": "block"}
),
html.Button(
"Sauver",
id="btn_save",
className="rounded border px-2 py-1 border-green-500 hover:bg-green-500",
style={"display": "none"}
),
html.Button(
"Annuler",
id="btn_cancel",
className="rounded border px-2 py-1 border-red-500 hover:bg-red-500",
style={"display": "none"}
),
],
className="flex flex-row space-x-2",
id="toolbar"
),
],
className="flex flex-row justify-between p-4"
),
html.Div([
html.Div([
dash_table.DataTable(
id="datatable",
data=df.to_dict('records'),
columns=[{"name": i, "id": i} for i in df.columns],
filter_action="native",
sort_action="native",
sort_mode="multi",
editable=False
)
])
],
className="overflow-y-auto"
),
],
className="p-2"
)
@callback(
Output("datatable", 'editable', allow_duplicate=True),
Output("table_backup", 'data'),
Input("btn_edit", "n_clicks"),
State("datatable", 'data'),
prevent_initial_call=True
)
def activate_editable(n_clicks, df_src):
if n_clicks is None:
raise PreventUpdate
if n_clicks > 0:
df_backup = df_src.copy()
return True, df_backup
raise PreventUpdate
@callback(
Output("datatable", 'editable'),
Output("datatable", 'data'),
Input("btn_cancel", "n_clicks"),
State("table_backup", 'data'),
prevent_initial_call=True
)
def cancel_modifications(n_clicks, data):
if n_clicks is None:
raise PreventUpdate
if n_clicks > 0 and data is not None:
return False, data.copy()
raise PreventUpdate
# @callback(
# Output("datatable", 'editable'),
# Input("btn_save", "n_clicks"),
# State("datatable", 'editable'),
# )
# def save_modifications(n_clicks, editable):
# if n_clicks is None:
# raise PreventUpdate
# if n_clicks > 0:
# return not editable
# return editable
@callback(
Output("btn_edit", "style"),
Output("btn_save", "style"),
Output("btn_cancel", "style"),
Input("datatable", "editable"),
)
def toolbar(editable):
if editable:
return {"display": "none"}, {"display": "block"}, {"display": "block"}
return {"display": "block"}, {"display": "none"}, {"display": "none"}