ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

version bump for parallel fix

Browse source
This commit is contained in:
Ari Brown 2023-08-18 13:44:41 -04:00
parent 2613866fa3
commit 073de9fb95
2 changed files with 6 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
minerva/athena.py
View file

@ -125,15 +125,17 @@ class Query(Execute):
# dataset of the results. # dataset of the results.
# Calls `self.manifest_files()` which blocks via `self.finish()` # Calls `self.manifest_files()` which blocks via `self.finish()`
def results(self): def results(self):
#local = [self.handler.s3.download(f) for f in self.manifest_files()] start = time.time()
local = parallel_map(self.handler.s3.download, self.manifest_files()) local = [self.handler.s3.download(f) for f in self.manifest_files()]
#local = parallel_map(self.handler.s3.download, self.manifest_files())
print(time.time() - start)
self.ds = pa.dataset.dataset(local) self.ds = pa.dataset.dataset(local)
return self.ds return self.ds
def __enter__(self): def __enter__(self):
return self return self
def __exit__(self): def __exit__(self, exception_type, exception_value, exception_traceback):
self.close() self.close()
def close(self): def close(self):

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "minerva" name = "minerva"
version = "0.3.3" version = "0.3.4"
description = "Easier access to AWS Athena and Redshift" description = "Easier access to AWS Athena and Redshift"
authors = [ authors = [
"Ari Brown <ari@airintech.com>", "Ari Brown <ari@airintech.com>",