feat: add schema and table listing

dashboard/app.py

@@ -1,7 +1,10 @@
 import dash
 from dash import Dash, html, dcc
+from .pages import home, config
 
 app = Dash(__name__, use_pages=True)
+dash.register_page(home.__name__, path='/', layout=home.layout)
+dash.register_page(config.__name__, path='/config', layout=config.layout)
 
 app.layout = html.Div([
     html.H1('Plesna'),

dashboard/datalake.py

@@ -0,0 +1,13 @@
+from .libs.stage.fs_stage import FSStage
+from dotenv import dotenv_values
+
+env = {
+    **dotenv_values(".env"),
+}
+
+stages = {
+    "raw": FSStage(f"{env['DATA_PATH']}/{env['RAW_SUBPATH']}"),
+    "staging": FSStage(f"{env['DATA_PATH']}/{env['STAGING_SUBPATH']}"),
+    "gold": FSStage(f"{env['DATA_PATH']}/{env['GOLD_SUBPATH']}"),
+    "mart": FSStage(f"{env['DATA_PATH']}/{env['MART_SUBPATH']}"),
+}

dashboard/libs/fs_schema.py

@@ -0,0 +1,35 @@
+from .schema import AbstractSchema
+from pathlib import Path
+
+class FSSchema(AbstractSchema):
+    def __init__(self, basepath, metadata_engine=None):
+        self.basepath = basepath
+        self._metadata_engine = metadata_engine
+
+    def ls(self, dir, only_files=True):
+        dirpath = Path(dir)
+        if only_files:
+            return [f for f in dirpath.iterdir() if f.is_dir()]
+        return [f for f in dirpath.iterdir()]
+
+    def tables(self, dir, only_files=True):
+        dirpath = Path(dir)
+        if only_files:
+            return [f for f in dirpath.iterdir() if f.is_dir()]
+        return [f for f in dirpath.iterdir()]
+
+    def info(self, path):
+        path = Path(path)
+        pass
+
+    def read(self, path):
+        path = Path(path)
+        pass
+
+    def write(self, path, content):
+        path = Path(path)
+        pass
+
+    def delete(self, path):
+        path = Path(path)
+        pass

dashboard/libs/stage/fs_stage.py

@@ -0,0 +1,48 @@
+from .stage import AbstractStage
+from pathlib import Path
+
+class FSStage(AbstractStage):
+    def __init__(self, basepath, metadata_engine=None):
+        self.basepath = Path(basepath)
+        self._metadata_engine = metadata_engine
+
+    def ls(self, dir, only_files=False, only_directories=False) -> list[str]:
+        dirpath = Path(dir)
+
+        if only_files:
+            return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if not f.is_dir()]
+
+        if only_directories:
+            return [str(f.relative_to(dirpath)) for f in dirpath.iterdir() if f.is_dir()]
+
+        return [str(f.relative_to(dirpath)) for f in dirpath.iterdir()]
+
+    def schemas(self) -> list[str]:
+        dirpath = self.basepath
+        return self.ls(dirpath, only_directories=True)
+
+    def tables(self, schema:str) -> list[str]:
+        dirpath = self.basepath / schema
+        return self.ls(dirpath, only_files=True)
+
+    def build_table_path(self, table:str, schema:str):
+        table_path = self.basepath
+        if schema == '':
+            return table_path / table
+        return table_path / schema / table
+
+    def info(self, table:str, schema:str=''):
+        table_path = self.build_table_path(table, schema)
+        pass
+
+    def read(self, table:str, schema:str=''):
+        table_path = self.build_table_path(table, schema)
+        pass
+
+    def write(self, table:str, content, schema:str=''):
+        table_path = self.build_table_path(table, schema)
+        pass
+
+    def delete(self, table:str, schema:str=''):
+        table_path = self.build_table_path(table, schema)
+        pass

dashboard/libs/stage/metadata.py

@@ -0,0 +1,5 @@
+from abc import ABC
+
+
+class AbstractMetadataEngine(ABC):
+    pass

dashboard/libs/stage/stage.py

@@ -0,0 +1,36 @@
+import abc
+from .metadata import AbstractMetadataEngine
+
+
+class AbstractStage(abc.ABC):
+    metadata_engine = AbstractMetadataEngine
+
+    @abc.abstractmethod
+    def schemas():
+        """ List schemas """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def tables(schema):
+        """ List table in schema"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def info(self, path):
+        """ Get infos about a file"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def read(self, path):
+        """ Get content of a file"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def write(self, path, content):
+        """ Write content into the file"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, path):
+        """ Delete the file """
+        raise NotImplementedError

dashboard/pages/config.py

@@ -1,4 +1,3 @@
-import dash
 from dash import html
 from dotenv import dotenv_values
 import os
@@ -9,8 +8,6 @@ env = {
 }
 
 
-dash.register_page(__name__, path='/config')
-
 layout = html.Div([
     html.H1('This is our Config page'),
     html.Ul(children = [html.Li(f"{k} = {v}") for k,v in env.items()]),

dashboard/pages/home.py

@@ -1,9 +1,51 @@
-import dash
 from dash import html
+from ..datalake import stages
+from ..libs.stage.stage import AbstractStage
+
+
+def html_list_schema(stage:AbstractStage, with_tables=True):
+    """ Build html list of schema in stage """
+    if with_tables:
+        return html.Ul(
+            [
+                html.Li(
+                    children = [
+                        html.Span(schema),
+                        html_list_table(stage, schema)
+                    ]
+                ) for schema in stage.schemas()
+            ]
+        )
+    return html.Ul(
+        [
+            html.Li(schema) for schema in stage.schemas()
+        ]
+    )
+
+
+
+def html_list_table(stage:AbstractStage, schema:str):
+    """ Build html list of table in stage """
+    return html.Ul(
+        [
+            html.Li(table) for table in stage.tables(schema=schema)
+        ]
+    )
+
 
-dash.register_page(__name__, path='/')
 
 layout = html.Div([
     html.H1('This is our Home page'),
-    html.Div('This is our Home page content.'),
+    html.Div(children=[
+        html.Ul(
+            children=[
+                html.Li(
+                    children=[
+                        html.Span(stagename),
+                        html_list_schema(stage)
+                    ]
+                ) for stagename, stage in stages.items()
+            ]
+        )
+    ]),
 ])