This commit is contained in:
Guillem Borrell 2024-05-21 21:11:47 +01:00
parent 1e881e7537
commit 1b747cb7a3
3 changed files with 22 additions and 7 deletions

View file

@ -6,6 +6,3 @@ pydantic-settings
s3fs
aiofiles
duckdb
polars
pyarrow
xlsx2csv

View file

@ -35,7 +35,7 @@ class DDB:
self.sheets = tuple(
self.db.query("select Field2 from metadata where Field1 = 'Sheets'")
.fetchall()[0][0]
.split(",")
.split(";")
)
self.path = path
@ -83,5 +83,18 @@ class DDB:
return self
def query(self, sql):
return self.db.query(sql)
def load_folder_local(self, path: str):
self.sheets = tuple(
self.query(
f"select Field2 from read_csv_auto('{path}/metadata.csv') where Field1 = 'Sheets'"
)
.fetchall()[0][0]
.split(";")
)
return self
def load_folder_gcs(self, path: str):
return self
def query(self, sql, *args, **kwargs):
return self.db.query(sql, *args, **kwargs)

View file

@ -6,7 +6,7 @@ TEST_DATA_FOLDER = Path(hellocomputer.__file__).parents[2] / "test" / "data"
TEST_OUTPUT_FOLDER = Path(hellocomputer.__file__).parents[2] / "test" / "output"
def test_load_data():
def test_dump():
db = (
DDB()
.load_metadata(TEST_DATA_FOLDER / "TestExcelHelloComputer.xlsx")
@ -15,3 +15,8 @@ def test_load_data():
assert db.sheets == ("answers",)
assert (TEST_OUTPUT_FOLDER / "answers.csv").exists()
def test_load():
db = DDB().load_folder_local(TEST_OUTPUT_FOLDER)
assert db.sheets == ("answers",)