ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

version bump for parallel fix

Browse source
This commit is contained in:
Ari Brown 2023-08-18 13:44:41 -04:00
parent 2613866fa3
commit 073de9fb95
2 changed files with 6 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
minerva/athena.py
View file

@ -125,15 +125,17 @@ class Query(Execute):
# dataset of the results.
# Calls `self.manifest_files()` which blocks via `self.finish()`
def results(self):
#local = [self.handler.s3.download(f) for f in self.manifest_files()]
local = parallel_map(self.handler.s3.download, self.manifest_files())
start = time.time()
local = [self.handler.s3.download(f) for f in self.manifest_files()]
#local = parallel_map(self.handler.s3.download, self.manifest_files())
print(time.time() - start)
self.ds = pa.dataset.dataset(local)
return self.ds
def __enter__(self):
return self
def __exit__(self):
def __exit__(self, exception_type, exception_value, exception_traceback):
self.close()
def close(self):