Also gcs support

This commit is contained in:
Guillem Borrell 2024-05-21 20:55:13 +01:00
parent ff781b6b9c
commit 1e881e7537
2 changed files with 20 additions and 9 deletions

View file

@ -43,7 +43,7 @@ class DDB:
def dump_local(self, path): def dump_local(self, path):
# TODO: Port to fsspec and have a single dump file # TODO: Port to fsspec and have a single dump file
self.db.query(f"copy (select * from metadata) to '{path}/metadata.csv'") self.db.query(f"copy metadata to '{path}/metadata.csv'")
for sheet in self.sheets: for sheet in self.sheets:
self.db.query(f""" self.db.query(f"""
@ -63,12 +63,24 @@ class DDB:
return self return self
def dump_gcs(self, bucketname, sid): def dump_gcs(self, bucketname, sid):
self.db.sql(f""" self.db.sql(f"copy metadata to 'gcs://{bucketname}/{sid}/data.csv'")
for sheet in self.sheets:
self.db.query(f"""
copy copy
data (
to select
'gcs://{bucketname}/{sid}/data.csv'; *
""") from
st_read
(
'{self.path}',
layer = '{sheet}'
)
)
to 'gcs://{bucketname}/{sid}/{sheet}.csv'
""")
return self return self
def query(self, sql): def query(self, sql):

View file

@ -31,8 +31,7 @@ async def upload_file(file: UploadFile = File(...), sid: str = ""):
DDB() DDB()
.gcs_secret(settings.gcs_secret, settings.gcs_secret) .gcs_secret(settings.gcs_secret, settings.gcs_secret)
.load_metadata(f.name) .load_metadata(f.name)
.load_data() .dump_gcs(settings.gcs_bucketname, sid)
.save_gcs(settings.gcs_bucketname, sid)
) )
return JSONResponse( return JSONResponse(