diff --git a/minerva/athena.py b/minerva/athena.py index 6c12e48..e6d927d 100644 --- a/minerva/athena.py +++ b/minerva/athena.py @@ -67,6 +67,8 @@ class Execute: time.sleep(5) stat = self.status() + ms = self.info_cache['Statistics']['TotalExecutionTimeInMillis'] + self.runtime = datetime.timedelta(seconds=ms / 1000) return stat # finalized state diff --git a/minerva/parallel.py b/minerva/parallel.py index 8e97ead..5f4a182 100644 --- a/minerva/parallel.py +++ b/minerva/parallel.py @@ -15,6 +15,10 @@ from joblib import Parallel, delayed # # parallel_map(say, [str(i) for i in range(10)], cores=4) def parallel_map(func=None, data=None, cores=8): + # Base case + if not data: + return [] + cores = min(cores, len(data)) size = math.ceil(len(data) / cores) groups = [data[i:i + size] for i in range(0, len(data), size)] diff --git a/pyproject.toml b/pyproject.toml index 7ffe039..5ee9448 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "minerva" -version = "0.3.2" +version = "0.3.3" description = "Easier access to AWS Athena and Redshift" authors = [ "Ari Brown ", @@ -15,3 +15,4 @@ readme = "README.md" python = ">3.9, <3.11" boto3 = "^1.26.161" pyarrow = "^12.0.1" +joblib = "^1.1.0"