bug fixes

This commit is contained in:
Ari Brown 2023-08-03 14:12:09 -04:00
parent dd2edecc64
commit 72fd7fe005
3 changed files with 8 additions and 1 deletions

View file

@ -67,6 +67,8 @@ class Execute:
time.sleep(5)
stat = self.status()
ms = self.info_cache['Statistics']['TotalExecutionTimeInMillis']
self.runtime = datetime.timedelta(seconds=ms / 1000)
return stat # finalized state

View file

@ -15,6 +15,10 @@ from joblib import Parallel, delayed
#
# parallel_map(say, [str(i) for i in range(10)], cores=4)
def parallel_map(func=None, data=None, cores=8):
# Base case
if not data:
return []
cores = min(cores, len(data))
size = math.ceil(len(data) / cores)
groups = [data[i:i + size] for i in range(0, len(data), size)]

View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "minerva"
version = "0.3.2"
version = "0.3.3"
description = "Easier access to AWS Athena and Redshift"
authors = [
"Ari Brown <ari@airintech.com>",
@ -15,3 +15,4 @@ readme = "README.md"
python = ">3.9, <3.11"
boto3 = "^1.26.161"
pyarrow = "^12.0.1"
joblib = "^1.1.0"