ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

bug fixes

Browse source
This commit is contained in:
Ari Brown 2023-08-03 14:12:09 -04:00
parent dd2edecc64
commit 72fd7fe005
3 changed files with 8 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
minerva/athena.py
View file

@ -67,6 +67,8 @@ class Execute:
time.sleep(5) time.sleep(5)
stat = self.status() stat = self.status()
ms = self.info_cache['Statistics']['TotalExecutionTimeInMillis']
self.runtime = datetime.timedelta(seconds=ms / 1000)
return stat # finalized state return stat # finalized state

4
minerva/parallel.py
View file

@ -15,6 +15,10 @@ from joblib import Parallel, delayed
# #
# parallel_map(say, [str(i) for i in range(10)], cores=4) # parallel_map(say, [str(i) for i in range(10)], cores=4)
def parallel_map(func=None, data=None, cores=8): def parallel_map(func=None, data=None, cores=8):
# Base case
if not data:
return []
cores = min(cores, len(data)) cores = min(cores, len(data))
size = math.ceil(len(data) / cores) size = math.ceil(len(data) / cores)
groups = [data[i:i + size] for i in range(0, len(data), size)] groups = [data[i:i + size] for i in range(0, len(data), size)]

3
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "minerva" name = "minerva"
version = "0.3.2" version = "0.3.3"
description = "Easier access to AWS Athena and Redshift" description = "Easier access to AWS Athena and Redshift"
authors = [ authors = [
"Ari Brown <ari@airintech.com>", "Ari Brown <ari@airintech.com>",
@ -15,3 +15,4 @@ readme = "README.md"
python = ">3.9, <3.11" python = ">3.9, <3.11"
boto3 = "^1.26.161" boto3 = "^1.26.161"
pyarrow = "^12.0.1" pyarrow = "^12.0.1"
joblib = "^1.1.0"