forked from bellwether/minerva
bug fixes
This commit is contained in:
parent
dd2edecc64
commit
72fd7fe005
3 changed files with 8 additions and 1 deletions
|
|
@ -67,6 +67,8 @@ class Execute:
|
|||
time.sleep(5)
|
||||
stat = self.status()
|
||||
|
||||
ms = self.info_cache['Statistics']['TotalExecutionTimeInMillis']
|
||||
self.runtime = datetime.timedelta(seconds=ms / 1000)
|
||||
return stat # finalized state
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ from joblib import Parallel, delayed
|
|||
#
|
||||
# parallel_map(say, [str(i) for i in range(10)], cores=4)
|
||||
def parallel_map(func=None, data=None, cores=8):
|
||||
# Base case
|
||||
if not data:
|
||||
return []
|
||||
|
||||
cores = min(cores, len(data))
|
||||
size = math.ceil(len(data) / cores)
|
||||
groups = [data[i:i + size] for i in range(0, len(data), size)]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "minerva"
|
||||
version = "0.3.2"
|
||||
version = "0.3.3"
|
||||
description = "Easier access to AWS Athena and Redshift"
|
||||
authors = [
|
||||
"Ari Brown <ari@airintech.com>",
|
||||
|
|
@ -15,3 +15,4 @@ readme = "README.md"
|
|||
python = ">3.9, <3.11"
|
||||
boto3 = "^1.26.161"
|
||||
pyarrow = "^12.0.1"
|
||||
joblib = "^1.1.0"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue