Feat: create datacatalogue with fs_datacatalogue

This commit is contained in:
2024-11-03 10:40:23 +01:00
parent aa1ead5435
commit 88795fdad3
11 changed files with 190 additions and 0 deletions

View File

Binary file not shown.

View File

@@ -0,0 +1,7 @@
Username;Identifier;First name;Last name
booker12;9012;Rachel;Booker
grey07;2070;Laura;Grey
johnson81;4081;Craig;Johnson
jenkins46;9346;Mary;Jenkins
smith79;5079;Jamie;Smith
1 Username Identifier First name Last name
2 booker12 9012 Rachel Booker
3 grey07 2070 Laura Grey
4 johnson81 4081 Craig Johnson
5 jenkins46 9346 Mary Jenkins
6 smith79 5079 Jamie Smith

View File

@@ -0,0 +1,72 @@
import shutil
from pathlib import Path
import pytest
from plesna.datastore.fs_datacatalogue import FSDataCatalogue
FIXTURE_DIR = Path(__file__).parent / Path("./fs_files/")
@pytest.fixture
def location(tmp_path):
loc = tmp_path
username_loc = loc / "username"
username_loc.mkdir()
salary_loc = loc / "salary"
salary_loc.mkdir()
example_src = FIXTURE_DIR
assert example_src.exists()
for f in example_src.glob("*"):
if "username" in str(f):
shutil.copy(f, username_loc)
else:
shutil.copy(f, salary_loc)
return loc
def test_init(location):
repo = FSDataCatalogue("example", location)
assert repo.ls() == [
"username",
"salary",
]
assert repo.ls(recursive=True) == [
"username",
"salary",
]
def test_list_schema(location):
repo = FSDataCatalogue("example", location)
assert {id: s.model_dump()["id"] for id, s in repo.schemas().items()} == {
".": ".",
"username": "username",
"salary": "salary",
}
assert {id: s.model_dump()["value"] for id, s in repo.schemas().items()} == {
".": ".",
"username": "username",
"salary": "salary",
}
assert {id: s.model_dump()["path"] for id, s in repo.schemas().items()} == {
".": Path("."),
"username": Path("username"),
"salary": Path("salary"),
}
def test_list_tables(location):
repo = FSDataCatalogue("example", location)
assert repo.tables() == {}
assert {id: t.model_dump()["value"] for id,t in repo.tables("username").items()} == {
"username.csv": "username.csv",
"username-password-recovery-code.xlsx": "username-password-recovery-code.xlsx",
"username-password-recovery-code.xls": "username-password-recovery-code.xls",
}
assert {id: t.model_dump()["value"] for id,t in repo.tables("salary").items()} == {
"salary.pdf": "salary.pdf",
}