ari/minerva
1
0
Fork 0
forked from bellwether/minerva
Code Pull requests Activity

fixed repartition

Browse source
This commit is contained in:
Ari Brown 2024-09-24 10:48:47 -04:00
parent e3055333b3
commit 5e604f07f7
4 changed files with 10 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

7
examples/repartition.py
View file

@ -38,7 +38,7 @@ def repartition(mach, agents):
# Prep the info for the docker container
variables = {"min_agent": min(agents),
"max_agent": max(agents),
"source": src_top_level,
"source": sorted_top,
"destination": dst_top_level,
"secondary_destination": None}
@ -54,7 +54,7 @@ def repartition(mach, agents):
#####################################
# Prep the work
# Find out how many hours there are in the dataset
pool_size = 5
pool_size = 1
objs = s.m.s3.ls(src_top_level + "year=")
hours = set(["s3://" + '/'.join([o.bucket_name, *o.key.split("/")[0:-1]])
@ -87,9 +87,6 @@ try:
# Second part: repartition
pool.run(repartition, data=groups)
import IPython
IPython.embed()
finally:
pool.terminate()