From c4e0b71a98e12ab9954aa1dd8c3e112a105d9d1b Mon Sep 17 00:00:00 2001 From: Ari Brown Date: Wed, 2 Aug 2023 15:18:24 -0400 Subject: [PATCH] slightly changed the parallel algorithm --- minerva/parallel.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/minerva/parallel.py b/minerva/parallel.py index bf268b1..df11ab8 100644 --- a/minerva/parallel.py +++ b/minerva/parallel.py @@ -1,5 +1,10 @@ +import math from joblib import Parallel, delayed +# If you have a list of 100 elements and want to process it with 8 cores, +# it will split it into 8 chunks (7 chunks of 13, 1 chunk of 9). `func` is +# then run serially on the contents within its own process +# # Instead of taking each object in the list and giving it its own thread, # this splits the list into `cores` groups and gives each group its own # thread, where the group is now processed in series within its thread. @@ -11,7 +16,7 @@ from joblib import Parallel, delayed # parallel_map(say, [str(i) for i in range(10)], cores=4) def parallel_map(func=None, data=None, cores=8): cores = min(cores, len(data)) - size = len(data) // cores + size = math.ceil(len(data) / cores) groups = [data[i:i + size] for i in range(0, len(data), size)] def wrapper_func(fs):