ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

add a check to support errors being raised before tempfiles are created

Browse source
This commit is contained in:
Ari Brown 2023-08-21 09:34:16 -04:00
parent 073de9fb95
commit 209f32a7de
2 changed files with 7 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

15
minerva/athena.py
View file

@ -112,8 +112,6 @@ class Query(Execute):
files = self.handler.s3.read(manif).split("\n")
files = [f.strip() for f in files if f.strip()] # filter empty
self.files = files
return files
else:
print("Error")
@ -125,11 +123,9 @@ class Query(Execute):
# dataset of the results.
# Calls `self.manifest_files()` which blocks via `self.finish()`
def results(self):
start = time.time()
local = [self.handler.s3.download(f) for f in self.manifest_files()]
#local = parallel_map(self.handler.s3.download, self.manifest_files())
print(time.time() - start)
self.ds = pa.dataset.dataset(local)
self.temps = [self.handler.s3.download(f) for f in self.manifest_files()]
#local = parallel_map(self.handler.s3.download, self.manifest_files())
self.ds = pa.dataset.dataset(self.temps)
return self.ds
def __enter__(self):
@ -139,6 +135,7 @@ class Query(Execute):
self.close()
def close(self):
for file in self.files:
os.remove(file)
if self.temps:
for file in self.temps:
os.remove(file)