lafrite/plesna
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feat: retrieve last log

Browse Source
This commit is contained in:
Bertrand Benjamin 2025-01-15 18:12:11 +01:00
parent eec3a13dbb
commit 2a387a1bc8
2 changed files with 41 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
plesna/storage/metadata_repository/fs_metadata_repository.py
View File

@ -21,8 +21,10 @@ class FSMetaDataRepository(MetaDataRepository):
""" """
EXECUTION_FILEMODEL = "{flux_id}_execution.csv" FILEMODEL = {
MODIFICATION_FILEMODEL = "{table_id}_modification.csv" "execution": "{flux_id}_execution.csv",
"modification": "{table_id}_modification.csv",
}
def __init__(self, basepath: str): def __init__(self, basepath: str):
super().__init__() super().__init__()
@ -36,7 +38,7 @@ class FSMetaDataRepository(MetaDataRepository):
def add_flux(self, flux_id: str) -> str: def add_flux(self, flux_id: str) -> str:
"""Get the flux metadata""" """Get the flux metadata"""
filepath = self._basepath / self.EXECUTION_FILEMODEL.format(flux_id=flux_id) filepath = self._basepath / self.FILEMODEL["execution"].format(flux_id=flux_id)
filepath.touch() filepath.touch()
with open(filepath, "a") as csvfile: with open(filepath, "a") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=ExecutionLog.model_fields.keys()) writer = csv.DictWriter(csvfile, fieldnames=ExecutionLog.model_fields.keys())
@ -45,7 +47,7 @@ class FSMetaDataRepository(MetaDataRepository):
def register_flux_execution(self, flux_id: str, dt: datetime, output: dict) -> ExecutionLog: def register_flux_execution(self, flux_id: str, dt: datetime, output: dict) -> ExecutionLog:
"""Get the flux metadata""" """Get the flux metadata"""
filepath = self._basepath / self.EXECUTION_FILEMODEL.format(flux_id=flux_id) filepath = self._basepath / self.FILEMODEL["execution"].format(flux_id=flux_id)
metadata_ = ExecutionLog(datetime=dt, output={"data": output}) metadata_ = ExecutionLog(datetime=dt, output={"data": output})
if not filepath.exists: if not filepath.exists:
raise FSMetaDataRepositoryError(f"The flux {flux_id} hasn't been added yet.") raise FSMetaDataRepositoryError(f"The flux {flux_id} hasn't been added yet.")
@ -57,12 +59,12 @@ class FSMetaDataRepository(MetaDataRepository):
return metadata_ return metadata_
def flux(self, flux_id: str) -> ExecutionLog: def flux(self, flux_id: str) -> ExecutionLog:
"""Get the flux metadata""" """Get the last flux log"""
raise NotImplementedError return max(self.flux_logs(flux_id), key=lambda l: l.datetime)
def flux_logs(self, flux_id: str) -> list[ExecutionLog]: def flux_logs(self, flux_id: str) -> list[ExecutionLog]:
"""Get the flux metadata""" """Get all flux logs"""
filepath = self._basepath / self.EXECUTION_FILEMODEL.format(flux_id=flux_id) filepath = self._basepath / self.FILEMODEL["execution"].format(flux_id=flux_id)
if not filepath.exists: if not filepath.exists:
raise FSMetaDataRepositoryError(f"The flux {flux_id} hasn't been added yet.") raise FSMetaDataRepositoryError(f"The flux {flux_id} hasn't been added yet.")
with open(filepath, "r") as csvfile: with open(filepath, "r") as csvfile:

34
tests/storage/test_fs_metadata_repository.py
View File

@ -30,7 +30,9 @@ def test_add_flux(location, metadata_repository):
flux_id = "my_flux" flux_id = "my_flux"
metadata_repository.add_flux(flux_id) metadata_repository.add_flux(flux_id)
metadata_filepath = location / metadata_repository.EXECUTION_FILEMODEL.format(flux_id=flux_id) metadata_filepath = location / metadata_repository.FILEMODEL["execution"].format(
flux_id=flux_id
)
assert metadata_filepath.exists() assert metadata_filepath.exists()
with open(metadata_filepath, "r") as csvfile: with open(metadata_filepath, "r") as csvfile:
@ -50,7 +52,9 @@ def test_register_flux_execution(location, metadata_repository):
}, },
) )
metadata_filepath = location / metadata_repository.EXECUTION_FILEMODEL.format(flux_id=flux_id) metadata_filepath = location / metadata_repository.FILEMODEL["execution"].format(
flux_id=flux_id
)
with open(metadata_filepath, "r") as csvfile: with open(metadata_filepath, "r") as csvfile:
content = csvfile.read() content = csvfile.read()
assert ( assert (
@ -58,7 +62,7 @@ def test_register_flux_execution(location, metadata_repository):
) )
def test_register_and_get_flux_execution(location, metadata_repository): def test_register_and_get_logs(metadata_repository):
flux_id = "my_flux" flux_id = "my_flux"
metadata_repository.add_flux(flux_id) metadata_repository.add_flux(flux_id)
@ -86,3 +90,27 @@ def test_register_and_get_flux_execution(location, metadata_repository):
output=FluxMetaData(data={"truc": "chose"}), output=FluxMetaData(data={"truc": "chose"}),
), ),
] ]
def test_register_and_get_last_log(metadata_repository):
flux_id = "my_flux"
metadata_repository.add_flux(flux_id)
metadata_repository.register_flux_execution(
flux_id,
datetime(2023, 3, 15, 14, 30),
output={"truc": "machin"},
)
metadata_repository.register_flux_execution(
flux_id,
datetime(2024, 3, 15, 14, 30),
output={
"truc": "chose",
},
)
logs = metadata_repository.flux(flux_id)
assert logs == ExecutionLog(
datetime=datetime(2024, 3, 15, 14, 30),
output=FluxMetaData(data={"truc": "chose"}),
)