ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

add a check to support errors being raised before tempfiles are created

Browse source
This commit is contained in:
Ari Brown 2023-08-21 09:34:16 -04:00
parent 073de9fb95
commit 209f32a7de
2 changed files with 7 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

11
minerva/athena.py
View file

@ -112,8 +112,6 @@ class Query(Execute):
files = self.handler.s3.read(manif).split("\n") files = self.handler.s3.read(manif).split("\n")
files = [f.strip() for f in files if f.strip()] # filter empty files = [f.strip() for f in files if f.strip()] # filter empty
self.files = files
return files return files
else: else:
print("Error") print("Error")
@ -125,11 +123,9 @@ class Query(Execute):
# dataset of the results. # dataset of the results.
# Calls `self.manifest_files()` which blocks via `self.finish()` # Calls `self.manifest_files()` which blocks via `self.finish()`
def results(self): def results(self):
start = time.time() self.temps = [self.handler.s3.download(f) for f in self.manifest_files()]
local = [self.handler.s3.download(f) for f in self.manifest_files()]
#local = parallel_map(self.handler.s3.download, self.manifest_files()) #local = parallel_map(self.handler.s3.download, self.manifest_files())
print(time.time() - start) self.ds = pa.dataset.dataset(self.temps)
self.ds = pa.dataset.dataset(local)
return self.ds return self.ds
def __enter__(self): def __enter__(self):
@ -139,6 +135,7 @@ class Query(Execute):
self.close() self.close()
def close(self): def close(self):
for file in self.files: if self.temps:
for file in self.temps:
os.remove(file) os.remove(file)

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "minerva" name = "minerva"
version = "0.3.4" version = "0.3.5"
description = "Easier access to AWS Athena and Redshift" description = "Easier access to AWS Athena and Redshift"
authors = [ authors = [
"Ari Brown <ari@airintech.com>", "Ari Brown <ari@airintech.com>",