diff --git a/src/hellocomputer/analytics.py b/src/hellocomputer/analytics.py index 4ecfc8e..b8eec56 100644 --- a/src/hellocomputer/analytics.py +++ b/src/hellocomputer/analytics.py @@ -43,7 +43,7 @@ class DDB: def dump_local(self, path): # TODO: Port to fsspec and have a single dump file - self.db.query(f"copy (select * from metadata) to '{path}/metadata.csv'") + self.db.query(f"copy metadata to '{path}/metadata.csv'") for sheet in self.sheets: self.db.query(f""" @@ -63,12 +63,24 @@ class DDB: return self def dump_gcs(self, bucketname, sid): - self.db.sql(f""" - copy - data - to - 'gcs://{bucketname}/{sid}/data.csv'; - """) + self.db.sql(f"copy metadata to 'gcs://{bucketname}/{sid}/data.csv'") + + for sheet in self.sheets: + self.db.query(f""" + copy + ( + select + * + from + st_read + ( + '{self.path}', + layer = '{sheet}' + ) + ) + to 'gcs://{bucketname}/{sid}/{sheet}.csv' + """) + return self def query(self, sql): diff --git a/src/hellocomputer/routers/files.py b/src/hellocomputer/routers/files.py index cdc7327..23b4d55 100644 --- a/src/hellocomputer/routers/files.py +++ b/src/hellocomputer/routers/files.py @@ -31,8 +31,7 @@ async def upload_file(file: UploadFile = File(...), sid: str = ""): DDB() .gcs_secret(settings.gcs_secret, settings.gcs_secret) .load_metadata(f.name) - .load_data() - .save_gcs(settings.gcs_bucketname, sid) + .dump_gcs(settings.gcs_bucketname, sid) ) return JSONResponse(