Feat: execute flux on dataplatform

tests/compute/test_consume_flux.py

@@ -7,18 +7,18 @@ from plesna.models.transformation import Transformation
 def test_consume_flux():
     sources = {
         "src1": Table(
-            id="src1", repo_id="test", schema_id="test", name="test", value="here"
+            id="src1", repo_id="test", schema_id="test", name="test", value="here", datas=["d"]
         ),
         "src2": Table(
-            id="src2", repo_id="test", schema_id="test", name="test", value="here"
+            id="src2", repo_id="test", schema_id="test", name="test", value="here", datas=["d"]
         ),
     }
     targets = {
         "tgt1": Table(
-            id="tgt1", repo_id="test", schema_id="test", name="test", value="this"
+            id="tgt1", repo_id="test", schema_id="test", name="test", value="this", datas=["d"]
         ),
         "tgt2": Table(
-            id="tgt2", repo_id="test", schema_id="test", name="test", value="that"
+            id="tgt2", repo_id="test", schema_id="test", name="test", value="that", datas=["d"]
         ),
     }
 

tests/dataplatform/test_dataplateform.py

@@ -29,7 +29,7 @@ def repository(tmp_path) -> FSRepository:
     for f in example_src.glob("*"):
         if "recovery" in str(f):
             shutil.copy(f, recovery_loc)
-        if "salary" in str(f):
+        elif "salary" in str(f):
             shutil.copy(f, salary_loc)
         else:
             shutil.copy(f, username_loc)
@@ -73,20 +73,23 @@ def foo_flux(repository: FSRepository) -> Flux:
 
 @pytest.fixture
 def copy_flux(repository: FSRepository) -> Flux:
-    src = {"username": repository.table("raw", "username")}
-    targets = {"username": repository.table("bronze", "username")}
+    raw_username = {"username": repository.table("raw", "username")}
+    bronze_username = {"username": repository.table("bronze", "username")}
 
     def copy(sources, targets):
-        pass
+        src_path = Path(sources["username"].datas[0])
+        tgt_path = Path(targets["username"].datas[0])
+        shutil.copy(src_path, tgt_path)
+        return {"src_size": src_path.stat().st_size, "tgt_size": tgt_path.stat().st_size}
 
     extra_kwrds = {}
 
-    flux = Flux(
-        sources=src,
-        targets=targets,
+    raw_brz_copy_username = Flux(
+        sources=raw_username,
+        targets=bronze_username,
         transformation=Transformation(function=copy, extra_kwrds=extra_kwrds),
     )
-    return flux
+    return raw_brz_copy_username
 
 
 def test_add_flux(repository: FSRepository, copy_flux: Flux):
@@ -113,7 +116,7 @@ def dataplatform(
     dp.add_repository("test", repository)
 
     dp.add_flux("foo", foo_flux)
-    dp.add_flux("copy", copy_flux)
+    dp.add_flux("raw_brz_copy_username", copy_flux)
     return dp
 
 
@@ -124,8 +127,16 @@ def test_listing_content(dataplatform: DataPlateform):
         "username",
         "salary",
     ]
+    assert dataplatform.repository("test").table("raw", "username").partitions == ["username.csv"]
 
 
 def test_execute_flux(dataplatform: DataPlateform):
     meta = dataplatform.execute_flux("foo")
     assert meta.data == {"who": "foo"}
+
+    assert dataplatform.repository("test").schema("bronze").tables == []
+
+    meta = dataplatform.execute_flux("raw_brz_copy_username")
+    assert meta.data == {"src_size": 175, "tgt_size": 175}
+
+    assert dataplatform.repository("test").schema("bronze").tables == ["username"]